Merge branch 'main' into 135678-unified-field-list-sections

x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx

@@ -85,6 +85,13 @@ export const PreviewHistogram = ({
     [timeframeOptions]
   );
   const endDate = useMemo(() => timeframeOptions.timeframeEnd.toISOString(), [timeframeOptions]);
+  // It seems like the Table/Grid component uses end date value as a non-inclusive one,
+  // thus the alerts which have timestamp equal to the end date value are not displayed in the table.
+  // To fix that, we extend end date value by 1s to make sure all alerts are included in the table.
+  const extendedEndDate = useMemo(
+    () => timeframeOptions.timeframeEnd.add('1', 's').toISOString(),
+    [timeframeOptions]
+  );
   const isEqlRule = useMemo(() => ruleType === 'eql', [ruleType]);
   const isMlRule = useMemo(() => ruleType === 'machine_learning', [ruleType]);
 
@@ -204,12 +211,8 @@ export const PreviewHistogram = ({
             columns,
             deletedEventIds,
             disabledCellActions: FIELDS_WITHOUT_CELL_ACTIONS,
-            // Fix for https://github.com/elastic/kibana/issues/135511, until we start writing proper
-            // simulated @timestamp values to the preview alerts. The preview alerts will have @timestamp values
-            // close to the server's `now` time, but the client clock could be out of sync with the server. So we
-            // avoid computing static dates for this time range filter and instead pass in a small relative time window.
-            end: 'now+5m',
-            start: 'now-5m',
+            end: extendedEndDate,
+            start: startDate,
             entityType: 'events',
             filters: [],
             globalFullScreen,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts

@@ -161,6 +161,7 @@ export const previewRulesRoute = async (
           ...securityRuleTypeOptions,
           ruleDataClient: previewRuleDataClient,
           ruleExecutionLoggerFactory: previewRuleExecutionLogger.factory,
+          isPreview: true,
         });
 
         const runExecutors = async <
@@ -406,7 +407,9 @@ export const previewRulesRoute = async (
             );
             break;
           case 'new_terms':
-            const newTermsAlertType = previewRuleTypeWrapper(createNewTermsAlertType(ruleOptions));
+            const newTermsAlertType = previewRuleTypeWrapper(
+              createNewTermsAlertType(ruleOptions, true)
+            );
             await runExecutors(
               newTermsAlertType.executor,
               newTermsAlertType.id,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts

@@ -46,7 +46,7 @@ import { buildTimestampRuntimeMapping } from './utils/build_timestamp_runtime_ma
 
 /* eslint-disable complexity */
 export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
-  ({ lists, logger, config, ruleDataClient, ruleExecutionLoggerFactory, version }) =>
+  ({ lists, logger, config, ruleDataClient, ruleExecutionLoggerFactory, version, isPreview }) =>
   (type) => {
     const { alertIgnoreFields: ignoreFields, alertMergeStrategy: mergeStrategy } = config;
     const persistenceRuleType = createPersistenceRuleTypeWrapper({ ruleDataClient, logger });
@@ -287,13 +287,15 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
               ruleExecutionLogger
             );
 
+            const alertTimestampOverride = isPreview ? startedAt : undefined;
             const legacySignalFields: string[] = Object.keys(aadFieldConversion);
             const wrapHits = wrapHitsFactory({
               ignoreFields: [...ignoreFields, ...legacySignalFields],
               mergeStrategy,
               completeRule,
               spaceId,
               indicesToQuery: inputIndex,
+              alertTimestampOverride,
             });
 
             const wrapSequences = wrapSequencesFactory({
@@ -303,6 +305,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
               completeRule,
               spaceId,
               indicesToQuery: inputIndex,
+              alertTimestampOverride,
             });
 
             const { filter: exceptionFilter, unprocessedExceptions } = await buildExceptionFilter({

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts

@@ -65,7 +65,8 @@ describe('buildAlert', () => {
         completeRule,
         SPACE_ID,
         reason,
-        completeRule.ruleParams.index as string[]
+        completeRule.ruleParams.index as string[],
+        undefined
       ),
       ...additionalAlertFields(doc),
     };
@@ -245,7 +246,8 @@ describe('buildAlert', () => {
         completeRule,
         SPACE_ID,
         reason,
-        completeRule.ruleParams.index as string[]
+        completeRule.ruleParams.index as string[],
+        undefined
       ),
       ...additionalAlertFields(doc),
     };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts

@@ -136,6 +136,7 @@ export const buildAlert = (
   spaceId: string | null | undefined,
   reason: string,
   indicesToQuery: string[],
+  alertTimestampOverride: Date | undefined,
   overrides?: {
     nameOverride: string;
     severityOverride: string;
@@ -179,7 +180,7 @@ export const buildAlert = (
   });
 
   return {
-    [TIMESTAMP]: new Date().toISOString(),
+    [TIMESTAMP]: alertTimestampOverride?.toISOString() ?? new Date().toISOString(),
     [SPACE_IDS]: spaceId != null ? [spaceId] : [],
     [EVENT_KIND]: 'signal',
     [ALERT_ORIGINAL_TIME]: originalTime?.toISOString(),

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts

@@ -59,7 +59,8 @@ describe('buildAlert', () => {
       'allFields',
       SPACE_ID,
       jest.fn(),
-      completeRule.ruleParams.index as string[]
+      completeRule.ruleParams.index as string[],
+      undefined
     );
     expect(alertGroup.length).toEqual(3);
     expect(alertGroup[0]).toEqual(

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts

@@ -43,7 +43,8 @@ export const buildAlertGroupFromSequence = (
   mergeStrategy: ConfigType['alertMergeStrategy'],
   spaceId: string | null | undefined,
   buildReasonMessage: BuildReasonMessage,
-  indicesToQuery: string[]
+  indicesToQuery: string[],
+  alertTimestampOverride: Date | undefined
 ): Array<WrappedFieldsLatest<EqlBuildingBlockFieldsLatest | EqlShellFieldsLatest>> => {
   const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event));
   if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) {
@@ -63,7 +64,8 @@ export const buildAlertGroupFromSequence = (
         [],
         false,
         buildReasonMessage,
-        indicesToQuery
+        indicesToQuery,
+        alertTimestampOverride
       )
     );
   } catch (error) {
@@ -93,7 +95,8 @@ export const buildAlertGroupFromSequence = (
     completeRule,
     spaceId,
     buildReasonMessage,
-    indicesToQuery
+    indicesToQuery,
+    alertTimestampOverride
   );
   const sequenceAlert: WrappedFieldsLatest<EqlShellFieldsLatest> = {
     _id: shellAlert[ALERT_UUID],
@@ -122,15 +125,23 @@ export const buildAlertRoot = (
   completeRule: CompleteRule<RuleParams>,
   spaceId: string | null | undefined,
   buildReasonMessage: BuildReasonMessage,
-  indicesToQuery: string[]
+  indicesToQuery: string[],
+  alertTimestampOverride: Date | undefined
 ): EqlShellFieldsLatest => {
   const mergedAlerts = objectArrayIntersection(wrappedBuildingBlocks.map((alert) => alert._source));
   const reason = buildReasonMessage({
     name: completeRule.ruleConfig.name,
     severity: completeRule.ruleParams.severity,
     mergedDoc: mergedAlerts as SignalSourceHit,
   });
-  const doc = buildAlert(wrappedBuildingBlocks, completeRule, spaceId, reason, indicesToQuery);
+  const doc = buildAlert(
+    wrappedBuildingBlocks,
+    completeRule,
+    spaceId,
+    reason,
+    indicesToQuery,
+    alertTimestampOverride
+  );
   const alertId = generateAlertId(doc);
   return {
     ...mergedAlerts,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts

@@ -51,7 +51,8 @@ export const buildBulkBody = (
   ignoreFields: ConfigType['alertIgnoreFields'],
   applyOverrides: boolean,
   buildReasonMessage: BuildReasonMessage,
-  indicesToQuery: string[]
+  indicesToQuery: string[],
+  alertTimestampOverride: Date | undefined
 ): BaseFieldsLatest => {
   const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields });
   const eventFields = buildEventTypeAlert(mergedDoc);
@@ -87,7 +88,15 @@ export const buildBulkBody = (
     return {
       ...filteredSource,
       ...eventFields,
-      ...buildAlert([mergedDoc], completeRule, spaceId, reason, indicesToQuery, overrides),
+      ...buildAlert(
+        [mergedDoc],
+        completeRule,
+        spaceId,
+        reason,
+        indicesToQuery,
+        alertTimestampOverride,
+        overrides
+      ),
       ...additionalAlertFields({ ...mergedDoc, _source: { ...mergedDoc._source, ...eventFields } }),
     };
   }

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts

@@ -22,6 +22,7 @@ describe('wrapNewTermsAlerts', () => {
       mergeStrategy: 'missingFields',
       completeRule,
       indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+      alertTimestampOverride: undefined,
     });
 
     expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58');
@@ -38,6 +39,7 @@ describe('wrapNewTermsAlerts', () => {
       mergeStrategy: 'missingFields',
       completeRule,
       indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+      alertTimestampOverride: undefined,
     });
 
     expect(alerts[0]._id).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545');
@@ -54,6 +56,7 @@ describe('wrapNewTermsAlerts', () => {
       mergeStrategy: 'missingFields',
       completeRule,
       indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+      alertTimestampOverride: undefined,
     });
 
     expect(alerts[0]._id).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea');
@@ -70,6 +73,7 @@ describe('wrapNewTermsAlerts', () => {
       mergeStrategy: 'missingFields',
       completeRule,
       indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+      alertTimestampOverride: undefined,
     });
 
     expect(alerts[0]._id).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce');

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts

@@ -31,12 +31,14 @@ export const wrapNewTermsAlerts = ({
   completeRule,
   mergeStrategy,
   indicesToQuery,
+  alertTimestampOverride,
 }: {
   eventsAndTerms: EventsAndTerms[];
   spaceId: string | null | undefined;
   completeRule: CompleteRule<RuleParams>;
   mergeStrategy: ConfigType['alertMergeStrategy'];
   indicesToQuery: string[];
+  alertTimestampOverride: Date | undefined;
 }): Array<WrappedFieldsLatest<NewTermsFieldsLatest>> => {
   return eventsAndTerms.map((eventAndTerms) => {
     const id = objectHash([
@@ -54,7 +56,8 @@ export const wrapNewTermsAlerts = ({
       [],
       true,
       buildReasonMessageForNewTermsAlert,
-      indicesToQuery
+      indicesToQuery,
+      alertTimestampOverride
     );
     return {
       _id: id,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts

@@ -26,12 +26,14 @@ export const wrapHitsFactory =
     mergeStrategy,
     spaceId,
     indicesToQuery,
+    alertTimestampOverride,
   }: {
     completeRule: CompleteRule<RuleParams>;
     ignoreFields: ConfigType['alertIgnoreFields'];
     mergeStrategy: ConfigType['alertMergeStrategy'];
     spaceId: string | null | undefined;
     indicesToQuery: string[];
+    alertTimestampOverride: Date | undefined;
   }) =>
   (
     events: Array<estypes.SearchHit<SignalSource>>,
@@ -56,7 +58,8 @@ export const wrapHitsFactory =
             ignoreFields,
             true,
             buildReasonMessage,
-            indicesToQuery
+            indicesToQuery,
+            alertTimestampOverride
           ),
           [ALERT_UUID]: id,
         },

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_sequences_factory.ts

@@ -24,13 +24,15 @@ export const wrapSequencesFactory =
     mergeStrategy,
     spaceId,
     indicesToQuery,
+    alertTimestampOverride,
   }: {
     logger: Logger;
     completeRule: CompleteRule<RuleParams>;
     ignoreFields: ConfigType['alertIgnoreFields'];
     mergeStrategy: ConfigType['alertMergeStrategy'];
     spaceId: string | null | undefined;
     indicesToQuery: string[];
+    alertTimestampOverride: Date | undefined;
   }): WrapSequences =>
   (sequences, buildReasonMessage) =>
     sequences.reduce(
@@ -43,7 +45,8 @@ export const wrapSequencesFactory =
           mergeStrategy,
           spaceId,
           buildReasonMessage,
-          indicesToQuery
+          indicesToQuery,
+          alertTimestampOverride
         ),
       ],
       []

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts

@@ -37,7 +37,8 @@ import {
 import { createEnrichEventsFunction } from '../../signals/enrichments';
 
 export const createNewTermsAlertType = (
-  createOptions: CreateRuleOptions
+  createOptions: CreateRuleOptions,
+  isPreview?: boolean
 ): SecurityAlertType<NewTermsRuleParams, {}, {}, 'default'> => {
   const { logger } = createOptions;
   return {
@@ -105,6 +106,7 @@ export const createNewTermsAlertType = (
         params,
         spaceId,
         state,
+        startedAt,
       } = execOptions;
 
       // Validate the history window size compared to `from` at runtime as well as in the `validate`
@@ -276,12 +278,14 @@ export const createNewTermsAlertType = (
             newTerms: [bucket.key],
           }));
 
+          const alertTimestampOverride = isPreview ? startedAt : undefined;
           const wrappedAlerts = wrapNewTermsAlerts({
             eventsAndTerms,
             spaceId,
             completeRule,
             mergeStrategy,
             indicesToQuery: inputIndex,
+            alertTimestampOverride,
           });
 
           const bulkCreateResult = await bulkCreate(

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -108,6 +108,7 @@ export interface CreateSecurityRuleTypeWrapperProps {
   ruleDataClient: IRuleDataClient;
   ruleExecutionLoggerFactory: IRuleExecutionLogService['createClientForExecutors'];
   version: string;
+  isPreview?: boolean;
 }
 
 export type CreateSecurityRuleTypeWrapper = (

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -104,6 +104,7 @@ describe('searchAfterAndBulkCreate', () => {
       ignoreFields: [],
       spaceId: 'default',
       indicesToQuery: inputIndexPattern,
+      alertTimestampOverride: undefined,
     });
   });
 

x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts

@@ -2121,7 +2121,7 @@ export const formattedPreviewStrategyResponse = {
                 aggs: {
                   preview: {
                     date_histogram: {
-                      field: 'signal.original_time',
+                      field: '@timestamp',
                       fixed_interval: '2700000ms',
                       min_doc_count: 0,
                       extended_bounds: { min: 1599574984482, max: 1599661384482 },
@@ -2157,7 +2157,7 @@ export const formattedPreviewStrategyResponse = {
                   },
                   {
                     range: {
-                      'signal.original_time': {
+                      '@timestamp': {
                         gte: '2020-09-08T14:23:04.482Z',
                         lte: '2020-09-09T14:23:04.482Z',
                         format: 'strict_date_optional_time',