Merge branch 'main' into feature/esql-validation

.buildkite/ftr_configs.yml

@@ -191,6 +191,7 @@ enabled:
   - x-pack/test/api_integration/apis/es/config.ts
   - x-pack/test/api_integration/apis/features/config.ts
   - x-pack/test/api_integration/apis/file_upload/config.ts
+  - x-pack/test/api_integration/apis/grok_debugger/config.ts
   - x-pack/test/api_integration/apis/kibana/config.ts
   - x-pack/test/api_integration/apis/lists/config.ts
   - x-pack/test/api_integration/apis/logs_ui/config.ts

.buildkite/pipeline-utils/buildkite/client.ts

@@ -339,12 +339,18 @@ export class BuildkiteClient {
   setAnnotation = (
     context: string,
     style: 'info' | 'success' | 'warning' | 'error',
-    value: string
+    value: string,
+    append: boolean = false
   ) => {
-    this.exec(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
-      input: value,
-      stdio: ['pipe', 'inherit', 'inherit'],
-    });
+    this.exec(
+      `buildkite-agent annotate --context '${context}' --style '${style}' ${
+        append ? '--append' : ''
+      }`,
+      {
+        input: value,
+        stdio: ['pipe', 'inherit', 'inherit'],
+      }
+    );
   };
 
   uploadArtifacts = (pattern: string) => {

.buildkite/pipelines/serverless_deployment/create_deployment_tag.yml

@@ -8,9 +8,11 @@ steps:
     commands:
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state initialize
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commits
-      - ts-node .buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts 25
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts 50
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_selection
     key: select_commit
+    env:
+      AUTO_SELECT_COMMIT: $AUTO_SELECT_COMMIT
 
   - wait: ~
 
@@ -21,6 +23,8 @@ steps:
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_confirmation
     key: collect_data
     depends_on: select_commit
+    env:
+      AUTO_SELECT_COMMIT: $AUTO_SELECT_COMMIT
 
   - wait: ~
 

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh

@@ -8,9 +8,14 @@ set -euo pipefail
 echo "--- Collecting commit info"
 ts-node .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts
 
-cat << EOF | buildkite-agent pipeline upload
+if [[ "$AUTO_SELECT_COMMIT" == "true" || "$AUTO_SELECT_COMMIT" == "1" ]]; then
+  echo "--- Auto promoting to RC, skipping confirmation"
+else
+  echo "--- Uploading confirmation step"
+  cat << EOF | buildkite-agent pipeline upload
   steps:
     - block: "Confirm deployment"
       prompt: "Are you sure you want to deploy to production? (dry run: ${DRY_RUN:-false})"
       depends_on: collect_data
 EOF
+fi

.buildkite/scripts/serverless/create_deploy_tag/info_sections/so_snapshot_comparison.ts

@@ -52,7 +52,7 @@ export function makeSOComparisonBlockHtml(comparisonResult: {
   if (comparisonResult.hasChanges) {
     return `<div>
 <h4>Plugin Saved Object migration changes: *yes, ${comparisonResult.changed.length} plugin(s)*</h4>
-<div>Changed plugins: ${comparisonResult.changed.join(', ')}</div>
+<div>Changed plugins: <strong>${comparisonResult.changed.join(', ')}</strong></div>
 <i>Find detailed info in the archived artifacts, or run the command yourself: </i>
 <div><pre>${comparisonResult.command}</pre></div>
 </div>`;

.buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts

@@ -6,13 +6,7 @@
  * Side Public License, v 1.
  */
 
-import {
-  buildkite,
-  COMMIT_INFO_CTX,
-  CommitWithStatuses,
-  exec,
-  SELECTED_COMMIT_META_KEY,
-} from './shared';
+import { buildkite, COMMIT_INFO_CTX, CommitWithStatuses, SELECTED_COMMIT_META_KEY } from './shared';
 import {
   getArtifactBuild,
   getOnMergePRBuild,
@@ -30,12 +24,33 @@ async function main(commitCountArg: string) {
 
   console.log('--- Updating buildkite context with listed commits');
   const commitListWithBuildResultsHtml = makeCommitInfoWithBuildResultsHtml(commitsWithStatuses);
-  exec(`buildkite-agent annotate --style 'info' --context '${COMMIT_INFO_CTX}'`, {
-    input: commitListWithBuildResultsHtml,
-  });
-
-  console.log('--- Generating buildkite input step');
-  addBuildkiteInputStep();
+  buildkite.setAnnotation(COMMIT_INFO_CTX, 'info', commitListWithBuildResultsHtml);
+
+  if (process.env.AUTO_SELECT_COMMIT?.match(/(1|true)/i)) {
+    console.log('--- Finding suitable candidate for auto-promotion');
+
+    const passingCommitCandidate = commitsWithStatuses.find((commit) => {
+      return (
+        commit.checks.onMergeBuild?.success &&
+        commit.checks.ftrBuild?.success &&
+        commit.checks.artifactBuild?.success
+      );
+    });
+
+    if (!passingCommitCandidate) {
+      throw new Error(
+        `Could not find a suitable candidate for auto-promotion in the last ${commitCount} commits. Stopping.`
+      );
+    }
+
+    console.log('Selected candidate: ', passingCommitCandidate);
+
+    console.log('--- Setting buildkite meta-data for auto-promotion');
+    buildkite.setMetadata(SELECTED_COMMIT_META_KEY, passingCommitCandidate.sha);
+  } else {
+    console.log('--- Generating buildkite input step');
+    addBuildkiteInputStep();
+  }
 }
 
 async function collectAvailableCommits(commitCount: number): Promise<GitCommitExtract[]> {
@@ -92,7 +107,7 @@ function addBuildkiteInputStep() {
     key: 'select-commit',
     fields: [
       {
-        text: 'Enter the release candidate commit SHA',
+        text: 'Enter the selected commit SHA',
         key: SELECTED_COMMIT_META_KEY,
       },
     ],

.buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts

@@ -11,6 +11,7 @@ import {
   COMMIT_INFO_CTX,
   CURRENT_COMMIT_META_KEY,
   DEPLOY_TAG_META_KEY,
+  DRY_RUN_CTX,
   octokit,
   SELECTED_COMMIT_META_KEY,
   sendSlackMessage,
@@ -21,6 +22,9 @@ import { getUsefulLinks } from './info_sections/useful_links';
 const WIZARD_CTX_INSTRUCTION = 'wizard-instruction';
 const WIZARD_CTX_DEFAULT = 'wizard-main';
 
+const IS_DRY_RUN = process.env.DRY_RUN?.match(/(1|true)/i);
+const IS_AUTOMATED_RUN = process.env.AUTO_SELECT_COMMIT?.match(/(1|true)/i);
+
 type StateNames =
   | 'start'
   | 'initialize'
@@ -40,6 +44,7 @@ interface StateShape {
   instruction?: string;
   instructionStyle?: 'success' | 'warning' | 'error' | 'info';
   display: boolean;
+  skipWhenAutomated?: boolean;
   pre?: (state: StateShape) => Promise<void | boolean>;
   post?: (state: StateShape) => Promise<void | boolean>;
 }
@@ -50,7 +55,15 @@ const states: Record<StateNames, StateShape> = {
     description: 'No description',
     display: false,
     post: async () => {
-      buildkite.setAnnotation(COMMIT_INFO_CTX, 'info', `<h4>:kibana: Release candidates</h4>`);
+      if (IS_DRY_RUN) {
+        buildkite.setAnnotation(
+          DRY_RUN_CTX,
+          'warning',
+          `Dry run: tag won't be pushed, slack won't be notified.`
+        );
+      }
+
+      buildkite.setAnnotation(COMMIT_INFO_CTX, 'info', `<h4>:kibana: Recent commits...</h4>`);
     },
   },
   initialize: {
@@ -69,30 +82,28 @@ const states: Record<StateNames, StateShape> = {
   },
   wait_for_selection: {
     name: 'Waiting for selection',
-    description: 'Waiting for the Release Manager to select a release candidate commit.',
+    description: 'Waiting for the Release Manager to select a commit.',
     instruction: `Please find, copy and enter a commit SHA to the buildkite input box to proceed.`,
     instructionStyle: 'warning',
+    skipWhenAutomated: true,
     display: true,
   },
   collect_commit_info: {
     name: 'Collecting commit info',
     description: 'Collecting supplementary info about the selected commit.',
-    instruction: `Please wait, while we're collecting data about the commit, and the release candidate.`,
+    instruction: `Please wait, while we're collecting data about the commits.`,
     instructionStyle: 'info',
     display: true,
     pre: async () => {
-      buildkite.setAnnotation(
-        COMMIT_INFO_CTX,
-        'info',
-        `<h4>:kibana: Selected release candidate info:</h4>`
-      );
+      buildkite.setAnnotation(COMMIT_INFO_CTX, 'info', `<h4>:kibana: Selected commit info:</h4>`);
     },
   },
   wait_for_confirmation: {
     name: 'Waiting for confirmation',
     description: 'Waiting for the Release Manager to confirm the release.',
     instruction: `Please review the collected information above and unblock the release on Buildkite, if you're satisfied.`,
     instructionStyle: 'warning',
+    skipWhenAutomated: true,
     display: true,
   },
   create_deploy_tag: {
@@ -227,10 +238,13 @@ export async function transition(targetStateName: StateNames, data?: any) {
 }
 
 function updateWizardState(stateData: Record<string, 'ok' | 'nok' | 'pending' | undefined>) {
-  const wizardHeader = `<h3>:kibana: Kibana Serverless deployment wizard :mage:</h3>`;
+  const wizardHeader = IS_AUTOMATED_RUN
+    ? `<h3>:kibana: Kibana Serverless automated promotion :robot_face:</h3>`
+    : `<h3>:kibana: Kibana Serverless deployment wizard :mage:</h3>`;
 
   const wizardSteps = Object.keys(states)
     .filter((stateName) => states[stateName].display)
+    .filter((stateName) => !(IS_AUTOMATED_RUN && states[stateName].skipWhenAutomated))
     .map((stateName) => {
       const stateInfo = states[stateName];
       const stateStatus = stateData[stateName];
@@ -259,7 +273,13 @@ ${wizardSteps.join('\n')}
 function updateWizardInstruction(targetState: string, stateData: any) {
   const { instructionStyle, instruction } = states[targetState];
 
-  if (instruction) {
+  if (IS_AUTOMATED_RUN) {
+    buildkite.setAnnotation(
+      WIZARD_CTX_INSTRUCTION,
+      'info',
+      `<i>It's an automated run, no action needed.</i>`
+    );
+  } else if (instruction) {
     buildkite.setAnnotation(
       WIZARD_CTX_INSTRUCTION,
       instructionStyle || 'info',
@@ -294,7 +314,6 @@ async function sendReleaseSlackAnnouncement({
   const textBlock = (...str: string[]) => ({ type: 'mrkdwn', text: str.join('\n') });
   const buildShortname = `kibana-serverless-release #${process.env.BUILDKITE_BUILD_NUMBER}`;
 
-  const isDryRun = process.env.DRY_RUN?.match('(1|true)');
   const mergedAtDate = targetCommitData.commit?.committer?.date;
   const mergedAtUtcString = mergedAtDate ? new Date(mergedAtDate).toUTCString() : 'unknown';
   const targetCommitSha = targetCommitData.sha;
@@ -309,19 +328,13 @@ async function sendReleaseSlackAnnouncement({
   ).data;
   const compareLink = currentCommitSha
     ? `<${compareResponse.html_url}|${compareResponse.total_commits} new commits>`
-    : 'a new release candidate';
+    : 'a new commit';
 
   const mainMessage = [
     `:ship_it_parrot: Promotion of ${compareLink} to QA has been <${process.env.BUILDKITE_BUILD_URL}|initiated>!\n`,
     `*Remember:* Promotion to Staging is currently a manual process and will proceed once the build is signed off in QA.\n`,
+    `cc: @kibana-serverless-promotion-notify`,
   ];
-  if (isDryRun) {
-    mainMessage.unshift(
-      `*:memo:This is a dry run - no commit will actually be promoted. Please ignore!*\n`
-    );
-  } else {
-    mainMessage.push(`cc: @kibana-serverless-promotion-notify`);
-  }
 
   const linksSection = {
     'Initiated by': process.env.BUILDKITE_BUILD_CREATOR || 'unknown',

.buildkite/scripts/serverless/create_deploy_tag/shared.ts

@@ -18,6 +18,7 @@ const CURRENT_COMMIT_META_KEY = 'current-commit-hash';
 
 const DEPLOY_TAG_META_KEY = 'deploy-tag';
 const COMMIT_INFO_CTX = 'commit-info';
+const DRY_RUN_CTX = 'dry-run';
 
 const octokit = getGithubClient();
 
@@ -51,6 +52,7 @@ export {
   COMMIT_INFO_CTX,
   DEPLOY_TAG_META_KEY,
   CURRENT_COMMIT_META_KEY,
+  DRY_RUN_CTX,
 };
 
 export interface CommitWithStatuses extends GitCommitExtract {
@@ -64,7 +66,29 @@ export interface CommitWithStatuses extends GitCommitExtract {
 }
 
 export function sendSlackMessage(payload: any) {
-  if (!process.env.DEPLOY_TAGGER_SLACK_WEBHOOK_URL) {
+  if (process.env.DRY_RUN?.match(/(1|true)/i)) {
+    const message =
+      typeof payload === 'string'
+        ? payload
+        : JSON.stringify(
+            payload,
+            // The slack playground doesn't like long strings
+            (_key, value) => (value?.length > 301 ? value.slice(0, 300) : value),
+            0
+          );
+    const slackPlaygroundLink = `https://app.slack.com/block-kit-builder/#${encodeURIComponent(
+      message
+    )}`;
+
+    buildkite.setAnnotation(
+      DRY_RUN_CTX,
+      'warning',
+      `Preview slack message <a href="${slackPlaygroundLink}">here</a>.`
+    );
+    console.log('DRY_RUN, not sending slack message:', slackPlaygroundLink);
+
+    return Promise.resolve();
+  } else if (!process.env.DEPLOY_TAGGER_SLACK_WEBHOOK_URL) {
     console.log('No SLACK_WEBHOOK_URL set, not sending slack message');
     return Promise.resolve();
   } else {

packages/kbn-management/settings/setting_ids/index.ts

@@ -167,6 +167,9 @@ export const SECURITY_SOLUTION_DEFAULT_ALERT_TAGS_KEY = 'securitySolution:alertT
 /** This Kibana Advanced Setting allows users to enable/disable the Expandable Flyout */
 export const SECURITY_SOLUTION_ENABLE_EXPANDABLE_FLYOUT_SETTING =
   'securitySolution:enableExpandableFlyout' as const;
+/** This Kibana Advanced Setting allows users to enable/disable querying cold and frozen data tiers in analyzer */
+export const SECURITY_SOLUTION_EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER =
+  'securitySolution:excludeColdAndFrozenTiersInAnalyzer' as const;
 
 // Timelion settings
 export const TIMELION_ES_DEFAULT_INDEX_ID = 'timelion:es.default_index';

packages/kbn-optimizer/limits.yml

@@ -63,7 +63,7 @@ pageLoadAssetSize:
   files: 22673
   filesManagement: 18683
   fileUpload: 25664
-  fleet: 158438
+  fleet: 174609
   globalSearch: 29696
   globalSearchBar: 50403
   globalSearchProviders: 25554