diff --git a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts index 2468df9df8237..320fd557bfb02 100644 --- a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts +++ b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts @@ -38,7 +38,7 @@ import { isRequestAbortedError } from '../lib/is_request_aborted_error'; import type { AiopsLicense } from '../types'; import { fetchChangePointPValues } from './queries/fetch_change_point_p_values'; -import { fetchFieldCandidates } from './queries/fetch_field_candidates'; +import { fetchIndexInfo } from './queries/fetch_index_info'; import { dropDuplicates, fetchFrequentItems, @@ -168,32 +168,44 @@ export const defineExplainLogRateSpikesRoute = ( logDebugMessage('Reset.'); push(resetAction()); pushPingWithTimeout(); - logDebugMessage('Load field candidates.'); + + // Step 1: Index Info: Field candidates, total doc count, sample probability + + const fieldCandidates: Awaited>['fieldCandidates'] = []; + let sampleProbability = 1; + let totalDocCount = 0; + + logDebugMessage('Fetch index information.'); push( updateLoadingStateAction({ ccsWarning: false, loaded, loadingState: i18n.translate( - 'xpack.aiops.explainLogRateSpikes.loadingState.loadingFieldCandidates', + 'xpack.aiops.explainLogRateSpikes.loadingState.loadingIndexInformation', { - defaultMessage: 'Loading field candidates.', + defaultMessage: 'Loading index information.', } ), }) ); - let fieldCandidates: Awaited>; try { - fieldCandidates = await fetchFieldCandidates(client, request.body, abortSignal); + const indexInfo = await fetchIndexInfo(client, request.body, abortSignal); + fieldCandidates.push(...indexInfo.fieldCandidates); + sampleProbability = indexInfo.sampleProbability; + totalDocCount = indexInfo.totalDocCount; } catch (e) { if (!isRequestAbortedError(e)) { - logger.error(`Failed to fetch field candidates, got: \n${e.toString()}`); - pushError(`Failed to fetch field candidates.`); + logger.error(`Failed to fetch index information, got: \n${e.toString()}`); + pushError(`Failed to fetch index information.`); } end(); return; } + logDebugMessage(`Total document count: ${totalDocCount}`); + logDebugMessage(`Sample probability: ${sampleProbability}`); + loaded += LOADED_FIELD_CANDIDATES; push( @@ -245,6 +257,7 @@ export const defineExplainLogRateSpikesRoute = ( request.body, fieldCandidatesChunk, logger, + sampleProbability, pushError, abortSignal ); @@ -396,6 +409,7 @@ export const defineExplainLogRateSpikesRoute = ( request.body.deviationMin, request.body.deviationMax, logger, + sampleProbability, pushError, abortSignal ); diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_change_point_p_values.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_change_point_p_values.ts index 08165db084670..6400cc08ca4db 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_change_point_p_values.ts +++ b/x-pack/plugins/aiops/server/routes/queries/fetch_change_point_p_values.ts @@ -10,6 +10,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import { ChangePoint } from '@kbn/ml-agg-utils'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { SPIKE_ANALYSIS_THRESHOLD } from '../../../common/constants'; import type { AiopsExplainLogRateSpikesSchema } from '../../../common/api/explain_log_rate_spikes'; @@ -23,7 +24,9 @@ import { getRequestBase } from './get_request_base'; export const getChangePointRequest = ( params: AiopsExplainLogRateSpikesSchema, - fieldName: string + fieldName: string, + // The default value of 1 means no sampling will be used + sampleProbability: number = 1 ): estypes.SearchRequest => { const query = getQueryWithParams({ params, @@ -50,36 +53,49 @@ export const getChangePointRequest = ( ]; } - const body = { - query, - size: 0, - aggs: { - change_point_p_value: { - significant_terms: { - field: fieldName, - background_filter: { - bool: { - filter: [ - ...filter, - { - range: { - [timeFieldName]: { - gte: params.baselineMin, - lt: params.baselineMax, - format: 'epoch_millis', - }, + const pValueAgg: Record = { + change_point_p_value: { + significant_terms: { + field: fieldName, + background_filter: { + bool: { + filter: [ + ...filter, + { + range: { + [timeFieldName]: { + gte: params.baselineMin, + lt: params.baselineMax, + format: 'epoch_millis', }, }, - ], - }, + }, + ], }, - p_value: { background_is_superset: false }, - size: 1000, }, + // @ts-expect-error `p_value` is not yet part of `AggregationsAggregationContainer` + p_value: { background_is_superset: false }, + size: 1000, }, }, }; + const randomSamplerAgg: Record = { + sample: { + // @ts-expect-error `random_sampler` is not yet part of `AggregationsAggregationContainer` + random_sampler: { + probability: sampleProbability, + }, + aggs: pValueAgg, + }, + }; + + const body = { + query, + size: 0, + aggs: sampleProbability < 1 ? randomSamplerAgg : pValueAgg, + }; + return { ...getRequestBase(params), body, @@ -92,11 +108,25 @@ interface Aggs extends estypes.AggregationsSignificantLongTermsAggregate { buckets: estypes.AggregationsSignificantLongTermsBucket[]; } +interface PValuesAggregation extends estypes.AggregationsSamplerAggregation { + change_point_p_value: Aggs; +} + +interface RandomSamplerAggregation { + sample: PValuesAggregation; +} + +function isRandomSamplerAggregation(arg: unknown): arg is RandomSamplerAggregation { + return isPopulatedObject(arg, ['sample']); +} + export const fetchChangePointPValues = async ( esClient: ElasticsearchClient, params: AiopsExplainLogRateSpikesSchema, fieldNames: string[], logger: Logger, + // The default value of 1 means no sampling will be used + sampleProbability: number = 1, emitError: (m: string) => void, abortSignal?: AbortSignal ): Promise => { @@ -104,12 +134,9 @@ export const fetchChangePointPValues = async ( const settledPromises = await Promise.allSettled( fieldNames.map((fieldName) => - esClient.search( - getChangePointRequest(params, fieldName), - { - signal: abortSignal, - maxRetries: 0, - } + esClient.search( + getChangePointRequest(params, fieldName, sampleProbability), + { signal: abortSignal, maxRetries: 0 } ) ) ); @@ -144,7 +171,9 @@ export const fetchChangePointPValues = async ( continue; } - const overallResult = resp.aggregations.change_point_p_value; + const overallResult = isRandomSamplerAggregation(resp.aggregations) + ? resp.aggregations.sample.change_point_p_value + : resp.aggregations.change_point_p_value; for (const bucket of overallResult.buckets) { const pValue = Math.exp(-bucket.score); diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts index 362cae07273e5..da4b8bbe5e792 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts +++ b/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts @@ -12,6 +12,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { Logger } from '@kbn/logging'; import type { ChangePoint, FieldValuePair } from '@kbn/ml-agg-utils'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; const FREQUENT_ITEMS_FIELDS_LIMIT = 15; @@ -21,6 +22,14 @@ interface FrequentItemsAggregation extends estypes.AggregationsSamplerAggregatio }; } +interface RandomSamplerAggregation { + sample: FrequentItemsAggregation; +} + +function isRandomSamplerAggregation(arg: unknown): arg is RandomSamplerAggregation { + return isPopulatedObject(arg, ['sample']); +} + export function dropDuplicates(cps: ChangePoint[], uniqueFields: Array) { return uniqWith(cps, (a, b) => isEqual(pick(a, uniqueFields), pick(b, uniqueFields))); } @@ -58,6 +67,8 @@ export async function fetchFrequentItems( deviationMin: number, deviationMax: number, logger: Logger, + // The default value of 1 means no sampling will be used + sampleProbability: number = 1, emitError: (m: string) => void, abortSignal?: AbortSignal ) { @@ -98,44 +109,40 @@ export async function fetchFrequentItems( field, })); - const totalDocCount = changePoints[0].total_doc_count; - const minDocCount = 50000; - let sampleProbability = 1; - - if (totalDocCount > minDocCount) { - sampleProbability = Math.min(0.5, minDocCount / totalDocCount); - } - - logger.debug(`frequent_items sample probability: ${sampleProbability}`); + const frequentItemsAgg: Record = { + fi: { + // @ts-expect-error `frequent_items` is not yet part of `AggregationsAggregationContainer` + frequent_items: { + minimum_set_size: 2, + size: 200, + minimum_support: 0.1, + fields: aggFields, + }, + }, + }; // frequent items can be slow, so sample and use 10% min_support - const aggs: Record = { + const randomSamplerAgg: Record = { sample: { + // @ts-expect-error `random_sampler` is not yet part of `AggregationsAggregationContainer` random_sampler: { probability: sampleProbability, }, - aggs: { - fi: { - // @ts-expect-error `frequent_items` is not yet part of `AggregationsAggregationContainer` - frequent_items: { - minimum_set_size: 2, - size: 200, - minimum_support: 0.1, - fields: aggFields, - }, - }, - }, + aggs: frequentItemsAgg, }, }; const esBody = { query, - aggs, + aggs: sampleProbability < 1 ? randomSamplerAgg : frequentItemsAgg, size: 0, track_total_hits: true, }; - const body = await client.search( + const body = await client.search< + unknown, + { sample: FrequentItemsAggregation } | FrequentItemsAggregation + >( { index, size: 0, @@ -156,13 +163,17 @@ export async function fetchFrequentItems( const totalDocCountFi = (body.hits.total as estypes.SearchTotalHits).value; - const shape = body.aggregations.sample.fi.buckets.length; + const frequentItems = isRandomSamplerAggregation(body.aggregations) + ? body.aggregations.sample.fi + : body.aggregations.fi; + + const shape = frequentItems.buckets.length; let maximum = shape; if (maximum > 50000) { maximum = 50000; } - const fiss = body.aggregations.sample.fi.buckets; + const fiss = frequentItems.buckets; fiss.length = maximum; const results: ItemsetResult[] = []; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_field_candidates.test.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.test.ts similarity index 86% rename from x-pack/plugins/aiops/server/routes/queries/fetch_field_candidates.test.ts rename to x-pack/plugins/aiops/server/routes/queries/fetch_index_info.test.ts index 53e21e7a9dcce..084c415a652cd 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_field_candidates.test.ts +++ b/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.test.ts @@ -11,7 +11,7 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import type { AiopsExplainLogRateSpikesSchema } from '../../../common/api/explain_log_rate_spikes'; -import { fetchFieldCandidates, getRandomDocsRequest } from './fetch_field_candidates'; +import { fetchIndexInfo, getRandomDocsRequest } from './fetch_index_info'; const params: AiopsExplainLogRateSpikesSchema = { index: 'the-index', @@ -26,7 +26,7 @@ const params: AiopsExplainLogRateSpikesSchema = { searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', }; -describe('query_field_candidates', () => { +describe('fetch_index_info', () => { describe('getRandomDocsRequest', () => { it('returns the most basic request body for a sample of random documents', () => { const req = getRandomDocsRequest(params); @@ -57,6 +57,7 @@ describe('query_field_candidates', () => { }, }, size: 1000, + track_total_hits: true, }, index: params.index, ignore_throttled: undefined, @@ -87,6 +88,7 @@ describe('query_field_candidates', () => { }, }, ], + total: { value: 5000000 }, }, } as unknown as estypes.SearchResponse; }); @@ -96,9 +98,14 @@ describe('query_field_candidates', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - const resp = await fetchFieldCandidates(esClientMock, params); + const { totalDocCount, sampleProbability, fieldCandidates } = await fetchIndexInfo( + esClientMock, + params + ); - expect(resp).toEqual(['myIpFieldName', 'myKeywordFieldName']); + expect(fieldCandidates).toEqual(['myIpFieldName', 'myKeywordFieldName']); + expect(sampleProbability).toEqual(0.01); + expect(totalDocCount).toEqual(5000000); expect(esClientFieldCapsMock).toHaveBeenCalledTimes(1); expect(esClientSearchMock).toHaveBeenCalledTimes(1); }); diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_field_candidates.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.ts similarity index 69% rename from x-pack/plugins/aiops/server/routes/queries/fetch_field_candidates.ts rename to x-pack/plugins/aiops/server/routes/queries/fetch_index_info.ts index 036d8c0f51fcf..f1444ef5972b2 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_field_candidates.ts +++ b/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.ts @@ -20,6 +20,7 @@ import { getRequestBase } from './get_request_base'; // `x-pack/plugins/apm/server/routes/correlations/queries/fetch_duration_field_candidates.ts` const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; +const SAMPLE_PROBABILITY_MIN_DOC_COUNT = 50000; const SUPPORTED_ES_FIELD_TYPES = [ ES_FIELD_TYPES.KEYWORD, @@ -42,14 +43,16 @@ export const getRandomDocsRequest = ( }, }, size: POPULATED_DOC_COUNT_SAMPLE_SIZE, + // Used to determine sample probability for follow up queries + track_total_hits: true, }, }); -export const fetchFieldCandidates = async ( +export const fetchIndexInfo = async ( esClient: ElasticsearchClient, params: AiopsExplainLogRateSpikesSchema, abortSignal?: AbortSignal -): Promise => { +): Promise<{ fieldCandidates: string[]; sampleProbability: number; totalDocCount: number }> => { const { index } = params; // Get all supported fields const respMapping = await esClient.fieldCaps( @@ -73,20 +76,32 @@ export const fetchFieldCandidates = async ( } }); - const resp = await esClient.search(getRandomDocsRequest(params), { - signal: abortSignal, - maxRetries: 0, - }); + // Only the deviation window will be used to identify field candidates and sample probability based on total doc count. + const resp = await esClient.search( + getRandomDocsRequest({ ...params, start: params.deviationMin, end: params.deviationMax }), + { + signal: abortSignal, + maxRetries: 0, + } + ); const sampledDocs = resp.hits.hits.map((d) => d.fields ?? {}); // Get all field names for each returned doc and flatten it - // to a list of unique field names used across all docs. - // and filter by list of acceptable fields and some APM specific unique fields. + // to a list of unique field names used across all docs + // and filter by list of acceptable fields. [...new Set(sampledDocs.map(Object.keys).flat(1))].forEach((field) => { if (acceptableFields.has(field)) { finalFieldCandidates.add(field); } }); - return [...finalFieldCandidates]; + const totalDocCount = (resp.hits.total as estypes.SearchTotalHits).value; + + let sampleProbability = 1; + + if (totalDocCount > SAMPLE_PROBABILITY_MIN_DOC_COUNT) { + sampleProbability = Math.min(0.5, SAMPLE_PROBABILITY_MIN_DOC_COUNT / totalDocCount); + } + + return { fieldCandidates: [...finalFieldCandidates], sampleProbability, totalDocCount }; }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx index a16482c043343..af47d8db9cb6a 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/preview_histogram.tsx @@ -85,6 +85,13 @@ export const PreviewHistogram = ({ [timeframeOptions] ); const endDate = useMemo(() => timeframeOptions.timeframeEnd.toISOString(), [timeframeOptions]); + // It seems like the Table/Grid component uses end date value as a non-inclusive one, + // thus the alerts which have timestamp equal to the end date value are not displayed in the table. + // To fix that, we extend end date value by 1s to make sure all alerts are included in the table. + const extendedEndDate = useMemo( + () => timeframeOptions.timeframeEnd.add('1', 's').toISOString(), + [timeframeOptions] + ); const isEqlRule = useMemo(() => ruleType === 'eql', [ruleType]); const isMlRule = useMemo(() => ruleType === 'machine_learning', [ruleType]); @@ -204,12 +211,8 @@ export const PreviewHistogram = ({ columns, deletedEventIds, disabledCellActions: FIELDS_WITHOUT_CELL_ACTIONS, - // Fix for https://github.com/elastic/kibana/issues/135511, until we start writing proper - // simulated @timestamp values to the preview alerts. The preview alerts will have @timestamp values - // close to the server's `now` time, but the client clock could be out of sync with the server. So we - // avoid computing static dates for this time range filter and instead pass in a small relative time window. - end: 'now+5m', - start: 'now-5m', + end: extendedEndDate, + start: startDate, entityType: 'events', filters: [], globalFullScreen, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts index 1c82dc1194cef..5c5094fae4633 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts @@ -161,6 +161,7 @@ export const previewRulesRoute = async ( ...securityRuleTypeOptions, ruleDataClient: previewRuleDataClient, ruleExecutionLoggerFactory: previewRuleExecutionLogger.factory, + isPreview: true, }); const runExecutors = async < @@ -406,7 +407,9 @@ export const previewRulesRoute = async ( ); break; case 'new_terms': - const newTermsAlertType = previewRuleTypeWrapper(createNewTermsAlertType(ruleOptions)); + const newTermsAlertType = previewRuleTypeWrapper( + createNewTermsAlertType(ruleOptions, true) + ); await runExecutors( newTermsAlertType.executor, newTermsAlertType.id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index bd630b1a71107..fabda51b5602d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -46,7 +46,7 @@ import { buildTimestampRuntimeMapping } from './utils/build_timestamp_runtime_ma /* eslint-disable complexity */ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = - ({ lists, logger, config, ruleDataClient, ruleExecutionLoggerFactory, version }) => + ({ lists, logger, config, ruleDataClient, ruleExecutionLoggerFactory, version, isPreview }) => (type) => { const { alertIgnoreFields: ignoreFields, alertMergeStrategy: mergeStrategy } = config; const persistenceRuleType = createPersistenceRuleTypeWrapper({ ruleDataClient, logger }); @@ -287,6 +287,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = ruleExecutionLogger ); + const alertTimestampOverride = isPreview ? startedAt : undefined; const legacySignalFields: string[] = Object.keys(aadFieldConversion); const wrapHits = wrapHitsFactory({ ignoreFields: [...ignoreFields, ...legacySignalFields], @@ -294,6 +295,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = completeRule, spaceId, indicesToQuery: inputIndex, + alertTimestampOverride, }); const wrapSequences = wrapSequencesFactory({ @@ -303,6 +305,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = completeRule, spaceId, indicesToQuery: inputIndex, + alertTimestampOverride, }); const { filter: exceptionFilter, unprocessedExceptions } = await buildExceptionFilter({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts index 5e6cccd11e037..685d85d40a6b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts @@ -65,7 +65,8 @@ describe('buildAlert', () => { completeRule, SPACE_ID, reason, - completeRule.ruleParams.index as string[] + completeRule.ruleParams.index as string[], + undefined ), ...additionalAlertFields(doc), }; @@ -245,7 +246,8 @@ describe('buildAlert', () => { completeRule, SPACE_ID, reason, - completeRule.ruleParams.index as string[] + completeRule.ruleParams.index as string[], + undefined ), ...additionalAlertFields(doc), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index c716af17361ec..cbc1d2cc89f96 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -136,6 +136,7 @@ export const buildAlert = ( spaceId: string | null | undefined, reason: string, indicesToQuery: string[], + alertTimestampOverride: Date | undefined, overrides?: { nameOverride: string; severityOverride: string; @@ -179,7 +180,7 @@ export const buildAlert = ( }); return { - [TIMESTAMP]: new Date().toISOString(), + [TIMESTAMP]: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), [SPACE_IDS]: spaceId != null ? [spaceId] : [], [EVENT_KIND]: 'signal', [ALERT_ORIGINAL_TIME]: originalTime?.toISOString(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts index e047c86e75473..6a10dac0637d5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts @@ -59,7 +59,8 @@ describe('buildAlert', () => { 'allFields', SPACE_ID, jest.fn(), - completeRule.ruleParams.index as string[] + completeRule.ruleParams.index as string[], + undefined ); expect(alertGroup.length).toEqual(3); expect(alertGroup[0]).toEqual( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts index 700986198468c..22728c8f248df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.ts @@ -43,7 +43,8 @@ export const buildAlertGroupFromSequence = ( mergeStrategy: ConfigType['alertMergeStrategy'], spaceId: string | null | undefined, buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[] + indicesToQuery: string[], + alertTimestampOverride: Date | undefined ): Array> => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { @@ -63,7 +64,8 @@ export const buildAlertGroupFromSequence = ( [], false, buildReasonMessage, - indicesToQuery + indicesToQuery, + alertTimestampOverride ) ); } catch (error) { @@ -93,7 +95,8 @@ export const buildAlertGroupFromSequence = ( completeRule, spaceId, buildReasonMessage, - indicesToQuery + indicesToQuery, + alertTimestampOverride ); const sequenceAlert: WrappedFieldsLatest = { _id: shellAlert[ALERT_UUID], @@ -122,7 +125,8 @@ export const buildAlertRoot = ( completeRule: CompleteRule, spaceId: string | null | undefined, buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[] + indicesToQuery: string[], + alertTimestampOverride: Date | undefined ): EqlShellFieldsLatest => { const mergedAlerts = objectArrayIntersection(wrappedBuildingBlocks.map((alert) => alert._source)); const reason = buildReasonMessage({ @@ -130,7 +134,14 @@ export const buildAlertRoot = ( severity: completeRule.ruleParams.severity, mergedDoc: mergedAlerts as SignalSourceHit, }); - const doc = buildAlert(wrappedBuildingBlocks, completeRule, spaceId, reason, indicesToQuery); + const doc = buildAlert( + wrappedBuildingBlocks, + completeRule, + spaceId, + reason, + indicesToQuery, + alertTimestampOverride + ); const alertId = generateAlertId(doc); return { ...mergedAlerts, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts index 4d63f5c848484..c887d51df3ee5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts @@ -51,7 +51,8 @@ export const buildBulkBody = ( ignoreFields: ConfigType['alertIgnoreFields'], applyOverrides: boolean, buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[] + indicesToQuery: string[], + alertTimestampOverride: Date | undefined ): BaseFieldsLatest => { const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields }); const eventFields = buildEventTypeAlert(mergedDoc); @@ -87,7 +88,15 @@ export const buildBulkBody = ( return { ...filteredSource, ...eventFields, - ...buildAlert([mergedDoc], completeRule, spaceId, reason, indicesToQuery, overrides), + ...buildAlert( + [mergedDoc], + completeRule, + spaceId, + reason, + indicesToQuery, + alertTimestampOverride, + overrides + ), ...additionalAlertFields({ ...mergedDoc, _source: { ...mergedDoc._source, ...eventFields } }), }; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts index 6742f21759648..cfa696d71c47f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts @@ -22,6 +22,7 @@ describe('wrapNewTermsAlerts', () => { mergeStrategy: 'missingFields', completeRule, indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + alertTimestampOverride: undefined, }); expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); @@ -38,6 +39,7 @@ describe('wrapNewTermsAlerts', () => { mergeStrategy: 'missingFields', completeRule, indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + alertTimestampOverride: undefined, }); expect(alerts[0]._id).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545'); @@ -54,6 +56,7 @@ describe('wrapNewTermsAlerts', () => { mergeStrategy: 'missingFields', completeRule, indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + alertTimestampOverride: undefined, }); expect(alerts[0]._id).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea'); @@ -70,6 +73,7 @@ describe('wrapNewTermsAlerts', () => { mergeStrategy: 'missingFields', completeRule, indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + alertTimestampOverride: undefined, }); expect(alerts[0]._id).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts index 38ae5b1be75ae..e39bcf67909ae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts @@ -31,12 +31,14 @@ export const wrapNewTermsAlerts = ({ completeRule, mergeStrategy, indicesToQuery, + alertTimestampOverride, }: { eventsAndTerms: EventsAndTerms[]; spaceId: string | null | undefined; completeRule: CompleteRule; mergeStrategy: ConfigType['alertMergeStrategy']; indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; }): Array> => { return eventsAndTerms.map((eventAndTerms) => { const id = objectHash([ @@ -54,7 +56,8 @@ export const wrapNewTermsAlerts = ({ [], true, buildReasonMessageForNewTermsAlert, - indicesToQuery + indicesToQuery, + alertTimestampOverride ); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 75d55a5756680..123374eeb32de 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -26,12 +26,14 @@ export const wrapHitsFactory = mergeStrategy, spaceId, indicesToQuery, + alertTimestampOverride, }: { completeRule: CompleteRule; ignoreFields: ConfigType['alertIgnoreFields']; mergeStrategy: ConfigType['alertMergeStrategy']; spaceId: string | null | undefined; indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; }) => ( events: Array>, @@ -56,7 +58,8 @@ export const wrapHitsFactory = ignoreFields, true, buildReasonMessage, - indicesToQuery + indicesToQuery, + alertTimestampOverride ), [ALERT_UUID]: id, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_sequences_factory.ts index b700c31cf7c72..d4de6c1537f48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_sequences_factory.ts @@ -24,6 +24,7 @@ export const wrapSequencesFactory = mergeStrategy, spaceId, indicesToQuery, + alertTimestampOverride, }: { logger: Logger; completeRule: CompleteRule; @@ -31,6 +32,7 @@ export const wrapSequencesFactory = mergeStrategy: ConfigType['alertMergeStrategy']; spaceId: string | null | undefined; indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; }): WrapSequences => (sequences, buildReasonMessage) => sequences.reduce( @@ -43,7 +45,8 @@ export const wrapSequencesFactory = mergeStrategy, spaceId, buildReasonMessage, - indicesToQuery + indicesToQuery, + alertTimestampOverride ), ], [] diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 630553bc4d78c..8fce33897bd0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -37,7 +37,8 @@ import { import { createEnrichEventsFunction } from '../../signals/enrichments'; export const createNewTermsAlertType = ( - createOptions: CreateRuleOptions + createOptions: CreateRuleOptions, + isPreview?: boolean ): SecurityAlertType => { const { logger } = createOptions; return { @@ -105,6 +106,7 @@ export const createNewTermsAlertType = ( params, spaceId, state, + startedAt, } = execOptions; // Validate the history window size compared to `from` at runtime as well as in the `validate` @@ -276,12 +278,14 @@ export const createNewTermsAlertType = ( newTerms: [bucket.key], })); + const alertTimestampOverride = isPreview ? startedAt : undefined; const wrappedAlerts = wrapNewTermsAlerts({ eventsAndTerms, spaceId, completeRule, mergeStrategy, indicesToQuery: inputIndex, + alertTimestampOverride, }); const bulkCreateResult = await bulkCreate( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 03634c19b0ba0..ffd9e587361e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -108,6 +108,7 @@ export interface CreateSecurityRuleTypeWrapperProps { ruleDataClient: IRuleDataClient; ruleExecutionLoggerFactory: IRuleExecutionLogService['createClientForExecutors']; version: string; + isPreview?: boolean; } export type CreateSecurityRuleTypeWrapper = ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 489237ea1b2c9..e49c86ae2de48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -104,6 +104,7 @@ describe('searchAfterAndBulkCreate', () => { ignoreFields: [], spaceId: 'default', indicesToQuery: inputIndexPattern, + alertTimestampOverride: undefined, }); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts index e45c68999acf7..5f0b69ba6dc87 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts @@ -2121,7 +2121,7 @@ export const formattedPreviewStrategyResponse = { aggs: { preview: { date_histogram: { - field: 'signal.original_time', + field: '@timestamp', fixed_interval: '2700000ms', min_doc_count: 0, extended_bounds: { min: 1599574984482, max: 1599661384482 }, @@ -2157,7 +2157,7 @@ export const formattedPreviewStrategyResponse = { }, { range: { - 'signal.original_time': { + '@timestamp': { gte: '2020-09-08T14:23:04.482Z', lte: '2020-09-09T14:23:04.482Z', format: 'strict_date_optional_time', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/__mocks__/index.ts index 12d46c4be2282..95f5bccb18303 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/__mocks__/index.ts @@ -33,7 +33,7 @@ export const expectedDsl = { aggs: { preview: { date_histogram: { - field: 'signal.original_time', + field: '@timestamp', fixed_interval: '2700000ms', min_doc_count: 0, extended_bounds: { min: 1599574984482, max: 1599661384482 }, @@ -69,7 +69,7 @@ export const expectedDsl = { }, { range: { - 'signal.original_time': { + '@timestamp': { gte: '2020-09-08T14:23:04.482Z', lte: '2020-09-09T14:23:04.482Z', format: 'strict_date_optional_time', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/query.preview_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/query.preview_histogram.dsl.ts index 1703b6839ba70..2854ee25f9c43 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/query.preview_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/preview/query.preview_histogram.dsl.ts @@ -7,6 +7,7 @@ import moment from 'moment'; +import { TIMESTAMP } from '@kbn/rule-data-utils'; import { createQueryFilterClauses, calculateTimeSeriesInterval, @@ -23,7 +24,7 @@ export const buildPreviewHistogramQuery = ({ ...createQueryFilterClauses(filterQuery), { range: { - 'signal.original_time': { + [TIMESTAMP]: { gte: from, lte: to, format: 'strict_date_optional_time', @@ -34,7 +35,7 @@ export const buildPreviewHistogramQuery = ({ const getHistogramAggregation = () => { const interval = calculateTimeSeriesInterval(from, to); - const histogramTimestampField = 'signal.original_time'; + const histogramTimestampField = TIMESTAMP; const dateHistogram = { date_histogram: { field: histogramTimestampField, diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index cc8593d323ac7..7d1b626c1c7e8 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -6732,7 +6732,6 @@ "xpack.aiops.dataGrid.field.documentCountChartSplit.seriesLabel": "autre compte du document", "xpack.aiops.documentCountContent.clearSelectionAriaLabel": "Effacer la sélection", "xpack.aiops.explainLogRateSpikes.loadingState.doneMessage": "Terminé.", - "xpack.aiops.explainLogRateSpikes.loadingState.loadingFieldCandidates": "Chargement des champs candidats.", "xpack.aiops.explainLogRateSpikes.loadingState.loadingHistogramData": "Chargement des données d’histogramme.", "xpack.aiops.explainLogRateSpikesPage.emptyPromptBody": "La fonction Expliquer les pics de taux de log identifie les combinaisons champ/valeur statistiquement significatives qui contribuent à un pic de taux de log.", "xpack.aiops.explainLogRateSpikesPage.emptyPromptTitle": "Cliquez sur un pic dans l'histogramme pour lancer l'analyse.", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 141e9ad25e223..d3f6cb84354ae 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -6726,7 +6726,6 @@ "xpack.aiops.dataGrid.field.documentCountChartSplit.seriesLabel": "他のドキュメントカウント", "xpack.aiops.documentCountContent.clearSelectionAriaLabel": "選択した項目をクリア", "xpack.aiops.explainLogRateSpikes.loadingState.doneMessage": "完了しました。", - "xpack.aiops.explainLogRateSpikes.loadingState.loadingFieldCandidates": "フィールド候補を読み込んでいます。", "xpack.aiops.explainLogRateSpikes.loadingState.loadingHistogramData": "ヒストグラムデータを読み込んでいます。", "xpack.aiops.explainLogRateSpikesPage.emptyPromptBody": "explainログレートスパイク機能は、ログレートのスパイクに寄与する統計的に有意なフィールド/値の組み合わせを特定します。", "xpack.aiops.explainLogRateSpikesPage.emptyPromptTitle": "ヒストグラム図のスパイクをクリックすると、分析が開始します。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index cf4ca4adb9042..1239010846ed6 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -6736,7 +6736,6 @@ "xpack.aiops.dataGrid.field.documentCountChartSplit.seriesLabel": "其他文档计数", "xpack.aiops.documentCountContent.clearSelectionAriaLabel": "清除所选内容", "xpack.aiops.explainLogRateSpikes.loadingState.doneMessage": "完成。", - "xpack.aiops.explainLogRateSpikes.loadingState.loadingFieldCandidates": "正在加载字段候选项。", "xpack.aiops.explainLogRateSpikes.loadingState.loadingHistogramData": "正在加载直方图数据。", "xpack.aiops.explainLogRateSpikesPage.emptyPromptBody": "“解释日志速率峰值”功能会从统计上识别有助于达到日志速率峰值的重要字段/值组合。", "xpack.aiops.explainLogRateSpikesPage.emptyPromptTitle": "单击直方图中的某个峰值可开始分析。", diff --git a/x-pack/test/api_integration/apis/aiops/explain_log_rate_spikes.ts b/x-pack/test/api_integration/apis/aiops/explain_log_rate_spikes.ts index 2804fc3fc3e0e..58fe4a3c67fd3 100644 --- a/x-pack/test/api_integration/apis/aiops/explain_log_rate_spikes.ts +++ b/x-pack/test/api_integration/apis/aiops/explain_log_rate_spikes.ts @@ -276,7 +276,7 @@ export default ({ getService }: FtrProviderContext) => { const errorActions = data.filter((d) => d.type === expected.errorFilter); expect(errorActions.length).to.be(1); - expect(errorActions[0].payload).to.be('Failed to fetch field candidates.'); + expect(errorActions[0].payload).to.be('Failed to fetch index information.'); }); }); };