Merge branch 'main' into renovate/main-minify

.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml

@@ -18,7 +18,7 @@ spec:
       description: Kibana container image artifact builds
     spec:
       env:
-        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-serverless-test-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
       branch_configuration: main

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -133,7 +133,6 @@ steps:
       env:
         SERVICE_COMMIT_HASH: "$GIT_ABBREV_COMMIT"
         SERVICE: kibana
-        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-serverless-test-alerts'
         REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/dev.yaml
         GPCTL_PROMOTE_DRY_RUN: ${DRY_RUN:-false}
 EOF

.github/CODEOWNERS

@@ -1467,6 +1467,9 @@ x-pack/test/api_integration/apis/management/index_management/inference_endpoints
 /x-pack/test/plugin_functional/test_suites/resolver/ @elastic/security-solution
 /x-pack/test/security_solution_api_integration @elastic/security-solution
 /x-pack/test/api_integration/apis/security_solution @elastic/security-solution
+/x-pack/test/functional/es_archives/auditbeat/default @elastic/security-solution
+/x-pack/test/functional/es_archives/auditbeat/hosts @elastic/security-solution
+
 #CC# /x-pack/plugins/security_solution/ @elastic/security-solution
 
 # Security Solution OpenAPI bundles
@@ -1598,6 +1601,11 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 /x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users @elastic/security-threat-hunting-explore
 
+/x-pack/test/functional/es_archives/auditbeat/overview @elastic/security-threat-hunting-explore
+/x-pack/test/functional/es_archives/auditbeat/users @elastic/security-threat-hunting-explore
+
+/x-pack/test/functional/es_archives/auditbeat/uncommon_processes @elastic/security-threat-hunting-explore
+
 ## Generative AI owner connectors
 # OpenAI
 /x-pack/plugins/stack_connectors/public/connector_types/openai @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra

package.json

@@ -110,7 +110,7 @@
     "@elastic/apm-rum": "^5.16.1",
     "@elastic/apm-rum-core": "^5.21.1",
     "@elastic/apm-rum-react": "^2.0.3",
-    "@elastic/charts": "67.0.0",
+    "@elastic/charts": "67.0.1",
     "@elastic/datemath": "5.0.3",
     "@elastic/ebt": "^1.1.1",
     "@elastic/ecs": "^8.11.1",

src/plugins/maps_ems/public/kibana_services.ts

@@ -32,7 +32,7 @@ export function getIsEnterprisePlus() {
 }
 
 export async function setLicensingPluginStart(licensingPlugin: LicensingPluginStart) {
-  const license = await licensingPlugin.refresh();
+  const license = await licensingPlugin.getLicense();
   updateLicenseState(license);
   licensingPlugin.license$.subscribe(updateLicenseState);
 }

src/plugins/maps_ems/server/plugin.ts

@@ -7,19 +7,19 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { ILicense, LicensingPluginSetup } from '@kbn/licensing-plugin/server';
-import { Plugin, PluginInitializerContext } from '@kbn/core-plugins-server';
-import { CoreSetup } from '@kbn/core-lifecycle-server';
-import { MapConfig } from './config';
+import type { ILicense, LicensingPluginStart } from '@kbn/licensing-plugin/server';
+import type { Plugin, PluginInitializerContext } from '@kbn/core-plugins-server';
+import type { CoreSetup } from '@kbn/core-lifecycle-server';
+import type { MapConfig } from './config';
 import { LICENSE_CHECK_ID, EMSSettings } from '../common';
 
 export interface MapsEmsPluginServerSetup {
   config: MapConfig;
   createEMSSettings: () => EMSSettings;
 }
 
-interface MapsEmsSetupServerDependencies {
-  licensing?: LicensingPluginSetup;
+interface MapsEmsStartServerDependencies {
+  licensing?: LicensingPluginStart;
 }
 
 export class MapsEmsPlugin implements Plugin<MapsEmsPluginServerSetup> {
@@ -29,22 +29,20 @@ export class MapsEmsPlugin implements Plugin<MapsEmsPluginServerSetup> {
     this._initializerContext = initializerContext;
   }
 
-  public setup(core: CoreSetup, plugins: MapsEmsSetupServerDependencies) {
+  public setup(core: CoreSetup<MapsEmsStartServerDependencies>) {
     const mapConfig = this._initializerContext.config.get();
 
     let isEnterprisePlus = false;
-    if (plugins.licensing) {
-      function updateLicenseState(license: ILicense) {
-        const enterprise = license.check(LICENSE_CHECK_ID, 'enterprise');
-        isEnterprisePlus = enterprise.state === 'valid';
-      }
-
-      plugins.licensing
-        .refresh()
-        .then(updateLicenseState)
-        .catch(() => {});
-      plugins.licensing.license$.subscribe(updateLicenseState);
+    function updateLicenseState(license: ILicense) {
+      const enterprise = license.check(LICENSE_CHECK_ID, 'enterprise');
+      isEnterprisePlus = enterprise.state === 'valid';
     }
+    core
+      .getStartServices()
+      .then(([_, { licensing }]) => {
+        licensing?.license$.subscribe(updateLicenseState);
+      })
+      .catch(() => {});
 
     return {
       config: mapConfig,

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts

@@ -81,4 +81,5 @@ export const ReadKnowledgeBaseResponse = z.object({
   is_setup_available: z.boolean().optional(),
   is_setup_in_progress: z.boolean().optional(),
   pipeline_exists: z.boolean().optional(),
+  security_labs_exists: z.boolean().optional(),
 });

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml

@@ -78,6 +78,8 @@ paths:
                     type: boolean
                   pipeline_exists:
                     type: boolean
+                  security_labs_exists:
+                    type: boolean
         400:
           description: Generic Error
           content:

x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_status.tsx

@@ -90,6 +90,7 @@ export const isKnowledgeBaseSetup = (kbStatus: ReadKnowledgeBaseResponse | undef
   return (
     (kbStatus?.elser_exists &&
       kbStatus?.esql_exists &&
+      kbStatus?.security_labs_exists &&
       kbStatus?.index_exists &&
       kbStatus?.pipeline_exists) ??
     false

x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx

@@ -63,7 +63,8 @@ export const useChatSend = ({
     kbStatus?.elser_exists &&
     kbStatus?.index_exists &&
     kbStatus?.pipeline_exists &&
-    kbStatus?.esql_exists;
+    kbStatus?.esql_exists &&
+    kbStatus?.security_labs_exists;
 
   // Handles sending latest user prompt to API
   const handleSendMessage = useCallback(

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.tsx

@@ -55,8 +55,13 @@ export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
     // Resource enabled state
     const isElserEnabled = kbStatus?.elser_exists ?? false;
     const isESQLEnabled = kbStatus?.esql_exists ?? false;
+    const isSecurityLabsEnabled = kbStatus?.security_labs_exists ?? false;
     const isKnowledgeBaseSetup =
-      (isElserEnabled && isESQLEnabled && kbStatus?.index_exists && kbStatus?.pipeline_exists) ??
+      (isElserEnabled &&
+        isESQLEnabled &&
+        isSecurityLabsEnabled &&
+        kbStatus?.index_exists &&
+        kbStatus?.pipeline_exists) ??
       false;
     const isSetupInProgress = kbStatus?.is_setup_in_progress ?? false;
     const isSetupAvailable = kbStatus?.is_setup_available ?? false;

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/helpers.ts

@@ -12,13 +12,15 @@ import {
 } from '@kbn/elastic-assistant-common';
 import { z } from '@kbn/zod';
 
-export const isEsqlSystemEntry = (
+export const isSystemEntry = (
   entry: KnowledgeBaseEntryResponse
 ): entry is KnowledgeBaseEntryResponse & {
   type: DocumentEntryType;
-  kbResource: 'esql';
+  kbResource: 'esql' | 'security_labs';
 } => {
-  return entry.type === DocumentEntryType.value && entry.kbResource === 'esql';
+  return (
+    entry.type === DocumentEntryType.value && ['esql', 'security_labs'].includes(entry.kbResource)
+  );
 };
 
 export const isKnowledgeBaseEntryCreateProps = (

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx

@@ -43,7 +43,7 @@ import { KnowledgeBaseSettings } from '../knowledge_base_settings';
 import { SetupKnowledgeBaseButton } from '../setup_knowledge_base_button';
 import { useDeleteKnowledgeBaseEntries } from '../../assistant/api/knowledge_base/entries/use_delete_knowledge_base_entries';
 import {
-  isEsqlSystemEntry,
+  isSystemEntry,
   isKnowledgeBaseEntryCreateProps,
   isKnowledgeBaseEntryResponse,
 } from './helpers';
@@ -152,13 +152,13 @@ export const KnowledgeBaseSettingsManagement: React.FC = React.memo(() => {
           openFlyout();
         },
         isDeleteEnabled: (entry: KnowledgeBaseEntryResponse) => {
-          return !isEsqlSystemEntry(entry);
+          return !isSystemEntry(entry);
         },
         onDeleteActionClicked: ({ id }: KnowledgeBaseEntryResponse) => {
           deleteEntry({ ids: [id] });
         },
         isEditEnabled: (entry: KnowledgeBaseEntryResponse) => {
-          return !isEsqlSystemEntry(entry);
+          return !isSystemEntry(entry);
         },
         onEditActionClicked: ({ id }: KnowledgeBaseEntryResponse) => {
           const entry = entries.data.find((e) => e.id === id);

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/use_knowledge_base_table.tsx

@@ -18,7 +18,7 @@ import { useAssistantContext } from '../../..';
 import * as i18n from './translations';
 import { BadgesColumn } from '../../assistant/common/components/assistant_settings_management/badges';
 import { useInlineActions } from '../../assistant/common/components/assistant_settings_management/inline_actions';
-import { isEsqlSystemEntry } from './helpers';
+import { isSystemEntry } from './helpers';
 
 export const useKnowledgeBaseTable = () => {
   const { currentUserAvatar } = useAssistantContext();
@@ -29,7 +29,7 @@ export const useKnowledgeBaseTable = () => {
       if (entry.kbResource === 'user') {
         return 'userAvatar';
       }
-      if (entry.kbResource === 'esql') {
+      if (['esql', 'security_labs'].includes(entry.kbResource)) {
         return 'logoElastic';
       }
       return 'visText';
@@ -83,8 +83,8 @@ export const useKnowledgeBaseTable = () => {
           render: (entry: KnowledgeBaseEntryResponse) => {
             // TODO: Look up user from `createdBy` id if privileges allow
             const userName = entry.users?.[0]?.name ?? 'Unknown';
-            const badgeItem = isEsqlSystemEntry(entry) ? 'Elastic' : userName;
-            const userImage = isEsqlSystemEntry(entry) ? (
+            const badgeItem = isSystemEntry(entry) ? 'Elastic' : userName;
+            const userImage = isSystemEntry(entry) ? (
               <EuiIcon
                 type={'logoElastic'}
                 css={css`
@@ -124,7 +124,7 @@ export const useKnowledgeBaseTable = () => {
         {
           name: i18n.COLUMN_ENTRIES,
           render: (entry: KnowledgeBaseEntryResponse) => {
-            return isEsqlSystemEntry(entry)
+            return isSystemEntry(entry)
               ? entry.text
               : entry.type === DocumentEntryType.value
               ? '1'

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/setup_knowledge_base_button.tsx

@@ -34,7 +34,8 @@ export const SetupKnowledgeBaseButton: React.FC<Props> = React.memo(({ display }
     kbStatus?.elser_exists &&
     kbStatus?.index_exists &&
     kbStatus?.pipeline_exists &&
-    kbStatus?.esql_exists;
+    kbStatus?.esql_exists &&
+    kbStatus?.security_labs_exists;
 
   const onInstallKnowledgeBase = useCallback(() => {
     setupKB(ESQL_RESOURCE);

x-pack/packages/ml/agg_utils/src/types.ts

@@ -166,6 +166,9 @@ export interface SignificantItem extends FieldValuePair {
 
   /** Indicates if the significant item is unique within a group. */
   unique?: boolean;
+
+  /** Calculates a numerical value based on bg_count and doc_count for which to sort log rate change */
+  logRateChangeSort?: number;
 }
 
 interface SignificantItemHistogramItemBase {

x-pack/packages/ml/aiops_log_rate_analysis/get_log_rate_change.ts

@@ -38,7 +38,10 @@ export function getLogRateChange(
         }
       );
 
-      return { message, factor: roundedFactor };
+      return {
+        message,
+        factor: roundedFactor,
+      };
     } else {
       return {
         message: i18n.translate(
@@ -66,7 +69,10 @@ export function getLogRateChange(
         }
       );
 
-      return { message, factor: roundedFactor };
+      return {
+        message,
+        factor: roundedFactor,
+      };
     } else {
       return {
         message: i18n.translate(

x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx

@@ -37,6 +37,7 @@ import {
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils';
+import { useStorage } from '@kbn/ml-local-storage';
 import { AIOPS_TELEMETRY_ID } from '@kbn/aiops-common/constants';
 import type { AiopsLogRateAnalysisSchema } from '@kbn/aiops-log-rate-analysis/api/schema';
 import type { AiopsLogRateAnalysisSchemaSignificantItem } from '@kbn/aiops-log-rate-analysis/api/schema_v3';
@@ -53,6 +54,11 @@ import {
   commonColumns,
   significantItemColumns,
 } from '../log_rate_analysis_results_table/use_columns';
+import {
+  AIOPS_LOG_RATE_ANALYSIS_RESULT_COLUMNS,
+  type AiOpsKey,
+  type AiOpsStorageMapped,
+} from '../../types/storage';
 
 import {
   getGroupTableItems,
@@ -187,11 +193,10 @@ export const LogRateAnalysisResults: FC<LogRateAnalysisResultsProps> = ({
   );
   const [shouldStart, setShouldStart] = useState(false);
   const [toggleIdSelected, setToggleIdSelected] = useState(resultsGroupedOffId);
-  const [skippedColumns, setSkippedColumns] = useState<ColumnNames[]>([
-    'p-value',
-    'Baseline rate',
-    'Deviation rate',
-  ]);
+  const [skippedColumns, setSkippedColumns] = useStorage<
+    AiOpsKey,
+    AiOpsStorageMapped<typeof AIOPS_LOG_RATE_ANALYSIS_RESULT_COLUMNS>
+  >(AIOPS_LOG_RATE_ANALYSIS_RESULT_COLUMNS, ['p-value', 'Baseline rate', 'Deviation rate']);
   // null is used as the uninitialized state to identify the first load.
   const [skippedFields, setSkippedFields] = useState<string[] | null>(null);
 

x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx

@@ -51,7 +51,15 @@ export const LogRateAnalysisResultsTable: FC<LogRateAnalysisResultsTableProps> =
   const euiTheme = useEuiTheme();
   const primaryBackgroundColor = useEuiBackgroundColor('primary');
 
-  const allSignificantItems = useAppSelector((s) => s.logRateAnalysisResults.significantItems);
+  const allItems = useAppSelector((s) => s.logRateAnalysisResults.significantItems);
+
+  const allSignificantItems = useMemo(() => {
+    return allItems.map((item) => ({
+      ...item,
+      logRateChangeSort:
+        item.bg_count > 0 ? item.doc_count / item.bg_count : Number.POSITIVE_INFINITY,
+    }));
+  }, [allItems]);
 
   const significantItems = useMemo(() => {
     if (!groupFilter) {
@@ -85,7 +93,6 @@ export const LogRateAnalysisResultsTable: FC<LogRateAnalysisResultsTableProps> =
   );
 
   const dispatch = useAppDispatch();
-
   const [pageIndex, setPageIndex] = useState(0);
   const [pageSize, setPageSize] = useState(10);
   const [sortField, setSortField] = useState<keyof SignificantItem>(
@@ -135,8 +142,8 @@ export const LogRateAnalysisResultsTable: FC<LogRateAnalysisResultsTableProps> =
     ];
     const sortDirections = [sortDirection];
 
-    // Only if the table is sorted by p-value, add a secondary sort by doc count.
-    if (sortField === 'pValue') {
+    // If the table is sorted by p-value or log rate change, add a secondary sort by doc count.
+    if (sortField === 'pValue' || sortField === 'logRateChangeSort') {
       sortIteratees.push((item: SignificantItem) => item.doc_count);
       sortDirections.push(sortDirection);
     }
@@ -242,12 +249,12 @@ export const LogRateAnalysisResultsTable: FC<LogRateAnalysisResultsTableProps> =
     <EuiBasicTable
       data-test-subj="aiopsLogRateAnalysisResultsTable"
       compressed
-      columns={columns}
       items={pageOfItems}
-      onChange={onChange}
+      columns={columns}
       pagination={pagination.totalItemCount > pagination.pageSize ? pagination : undefined}
-      loading={false}
       sorting={sorting as EuiTableSortingType<SignificantItem>}
+      loading={false}
+      onChange={onChange}
       rowProps={(significantItem) => {
         return {
           'data-test-subj': `aiopsLogRateAnalysisResultsTableRow row-${significantItem.fieldName}-${significantItem.fieldValue}`,