From 7847bc8fe6eed1497af4ca57d3291eabe4588aee Mon Sep 17 00:00:00 2001 From: Dmitry Shevchenko Date: Thu, 16 Dec 2021 14:44:42 +0100 Subject: [PATCH] Test APM instrumentation of rule executors (#117672) --- .../rule_execution_log_client.ts | 38 +- .../rule_status_saved_objects_client.ts | 133 ++-- .../create_security_rule_type_wrapper.ts | 627 +++++++++--------- .../signals/bulk_create_factory.ts | 11 +- .../detection_engine/signals/executors/eql.ts | 146 ++-- .../detection_engine/signals/executors/ml.ts | 164 ++--- .../signals/executors/query.ts | 69 +- .../signals/executors/threat_match.ts | 72 +- .../signals/executors/threshold.ts | 239 +++---- .../detection_engine/signals/get_filter.ts | 6 +- .../signals/get_input_output_index.ts | 9 +- .../signals/search_after_bulk_create.ts | 267 ++++---- .../signals/signal_rule_alert_type.ts | 6 +- .../signals/single_search_after.ts | 115 ++-- .../lib/detection_engine/signals/utils.ts | 34 +- .../server/utils/with_security_span.ts | 36 + 16 files changed, 1038 insertions(+), 934 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/utils/with_security_span.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts index f3321580aa052..3efddf4d7afb0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts @@ -23,6 +23,7 @@ import { ExtMeta, } from './types'; import { truncateMessage } from './utils/normalization'; +import { withSecuritySpan } from '../../../utils/with_security_span'; interface ConstructorParams { underlyingClient: UnderlyingLogClient; @@ -56,30 +57,38 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient { /** @deprecated */ public find(args: FindExecutionLogArgs) { - return this.client.find(args); + return withSecuritySpan('RuleExecutionLogClient.find', () => this.client.find(args)); } /** @deprecated */ public findBulk(args: FindBulkExecutionLogArgs) { - return this.client.findBulk(args); + return withSecuritySpan('RuleExecutionLogClient.findBulk', () => this.client.findBulk(args)); } public getLastFailures(args: GetLastFailuresArgs): Promise { - return this.client.getLastFailures(args); + return withSecuritySpan('RuleExecutionLogClient.getLastFailures', () => + this.client.getLastFailures(args) + ); } public getCurrentStatus( args: GetCurrentStatusArgs ): Promise { - return this.client.getCurrentStatus(args); + return withSecuritySpan('RuleExecutionLogClient.getCurrentStatus', () => + this.client.getCurrentStatus(args) + ); } public getCurrentStatusBulk(args: GetCurrentStatusBulkArgs): Promise { - return this.client.getCurrentStatusBulk(args); + return withSecuritySpan('RuleExecutionLogClient.getCurrentStatusBulk', () => + this.client.getCurrentStatusBulk(args) + ); } - public deleteCurrentStatus(ruleId: string): Promise { - return this.client.deleteCurrentStatus(ruleId); + public async deleteCurrentStatus(ruleId: string): Promise { + await withSecuritySpan('RuleExecutionLogClient.deleteCurrentStatus', () => + this.client.deleteCurrentStatus(ruleId) + ); } public async logStatusChange(args: LogStatusChangeArgs): Promise { @@ -87,10 +96,17 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient { try { const truncatedMessage = message ? truncateMessage(message) : message; - await this.client.logStatusChange({ - ...args, - message: truncatedMessage, - }); + await withSecuritySpan( + { + name: 'RuleExecutionLogClient.logStatusChange', + labels: { new_rule_execution_status: args.newStatus }, + }, + () => + this.client.logStatusChange({ + ...args, + message: truncatedMessage, + }) + ); } catch (e) { const logMessage = 'Error logging rule execution status change'; const logAttributes = `status: "${newStatus}", rule id: "${ruleId}", rule name: "${ruleName}"`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts index cd26ab82b494a..5ec17e47a8b31 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts @@ -15,6 +15,7 @@ import { SavedObjectsUpdateResponse, } from 'kibana/server'; import { get } from 'lodash'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; // eslint-disable-next-line no-restricted-imports import { legacyRuleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; import { IRuleStatusSOAttributes } from '../../rules/types'; @@ -47,54 +48,58 @@ export const ruleStatusSavedObjectsClientFactory = ( savedObjectsClient: SavedObjectsClientContract ): RuleStatusSavedObjectsClient => ({ find: async (options) => { - const references = { - id: options.ruleId, - type: 'alert', - }; - const result = await savedObjectsClient.find({ - ...options, - type: legacyRuleStatusSavedObjectType, - hasReference: references, + return withSecuritySpan('RuleStatusSavedObjectsClient.find', async () => { + const references = { + id: options.ruleId, + type: 'alert', + }; + const result = await savedObjectsClient.find({ + ...options, + type: legacyRuleStatusSavedObjectType, + hasReference: references, + }); + return result.saved_objects; }); - return result.saved_objects; }, findBulk: async (ids, statusesPerId) => { if (ids.length === 0) { return {}; } - const references = ids.map((alertId) => ({ - id: alertId, - type: 'alert', - })); - const order: 'desc' = 'desc'; - // NOTE: Once https://github.com/elastic/kibana/issues/115153 is resolved - // ${legacyRuleStatusSavedObjectType}.statusDate will need to be updated to - // ${legacyRuleStatusSavedObjectType}.attributes.statusDate - const aggs = { - references: { - nested: { - path: `${legacyRuleStatusSavedObjectType}.references`, - }, - aggs: { - alertIds: { - terms: { - field: `${legacyRuleStatusSavedObjectType}.references.id`, - size: ids.length, - }, - aggs: { - rule_status: { - reverse_nested: {}, - aggs: { - most_recent_statuses: { - top_hits: { - sort: [ - { - [`${legacyRuleStatusSavedObjectType}.statusDate`]: { - order, + return withSecuritySpan('RuleStatusSavedObjectsClient.findBulk', async () => { + const references = ids.map((alertId) => ({ + id: alertId, + type: 'alert', + })); + const order: 'desc' = 'desc'; + // NOTE: Once https://github.com/elastic/kibana/issues/115153 is resolved + // ${legacyRuleStatusSavedObjectType}.statusDate will need to be updated to + // ${legacyRuleStatusSavedObjectType}.attributes.statusDate + const aggs = { + references: { + nested: { + path: `${legacyRuleStatusSavedObjectType}.references`, + }, + aggs: { + alertIds: { + terms: { + field: `${legacyRuleStatusSavedObjectType}.references.id`, + size: ids.length, + }, + aggs: { + rule_status: { + reverse_nested: {}, + aggs: { + most_recent_statuses: { + top_hits: { + sort: [ + { + [`${legacyRuleStatusSavedObjectType}.statusDate`]: { + order, + }, }, - }, - ], - size: statusesPerId, + ], + size: statusesPerId, + }, }, }, }, @@ -102,27 +107,33 @@ export const ruleStatusSavedObjectsClientFactory = ( }, }, }, - }, - }; - const results = await savedObjectsClient.find({ - hasReference: references, - aggs, - type: legacyRuleStatusSavedObjectType, - perPage: 0, + }; + const results = await savedObjectsClient.find({ + hasReference: references, + aggs, + type: legacyRuleStatusSavedObjectType, + perPage: 0, + }); + const buckets = get(results, 'aggregations.references.alertIds.buckets'); + return buckets.reduce((acc: Record, bucket: unknown) => { + const key = get(bucket, 'key'); + const hits = get(bucket, 'rule_status.most_recent_statuses.hits.hits'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + acc[key] = hits.map((hit: any) => hit._source[legacyRuleStatusSavedObjectType]); + return acc; + }, {}); }); - const buckets = get(results, 'aggregations.references.alertIds.buckets'); - return buckets.reduce((acc: Record, bucket: unknown) => { - const key = get(bucket, 'key'); - const hits = get(bucket, 'rule_status.most_recent_statuses.hits.hits'); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - acc[key] = hits.map((hit: any) => hit._source[legacyRuleStatusSavedObjectType]); - return acc; - }, {}); - }, - create: (attributes, options) => { - return savedObjectsClient.create(legacyRuleStatusSavedObjectType, attributes, options); }, + create: (attributes, options) => + withSecuritySpan('RuleStatusSavedObjectsClient.create', () => + savedObjectsClient.create(legacyRuleStatusSavedObjectType, attributes, options) + ), update: (id, attributes, options) => - savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes, options), - delete: (id) => savedObjectsClient.delete(legacyRuleStatusSavedObjectType, id), + withSecuritySpan('RuleStatusSavedObjectsClient.update', () => + savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes, options) + ), + delete: (id) => + withSecuritySpan('RuleStatusSavedObjectsClient.delete', () => + savedObjectsClient.delete(legacyRuleStatusSavedObjectType, id) + ), }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 7838e68b16161..68b48ec169667 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -9,6 +9,7 @@ import { isEmpty } from 'lodash'; import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; import { ListArray } from '@kbn/securitysolution-io-ts-list-types'; +import agent from 'elastic-apm-node'; import { createPersistenceRuleTypeWrapper } from '../../../../../rule_registry/server'; import { buildRuleMessageFactory } from './factories/build_rule_message_factory'; @@ -39,6 +40,7 @@ import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; import aadFieldConversion from '../routes/index/signal_aad_mapping.json'; import { extractReferences, injectReferences } from '../signals/saved_object_references'; +import { withSecuritySpan } from '../../../utils/with_security_span'; /* eslint-disable complexity */ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = @@ -54,273 +56,335 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = injectReferences({ logger, params, savedObjectReferences }), }, async executor(options) { - const { - alertId, - params, - previousStartedAt, - startedAt, - services, - spaceId, - state, - updatedBy: updatedByUser, - rule, - } = options; - let runState = state; - const { from, maxSignals, meta, ruleId, timestampOverride, to } = params; - const { alertWithPersistence, savedObjectsClient, scopedClusterClient } = services; - const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); - - const esClient = scopedClusterClient.asCurrentUser; - - const ruleStatusClient = ruleExecutionLogClientOverride - ? ruleExecutionLogClientOverride - : new RuleExecutionLogClient({ - underlyingClient: config.ruleExecutionLog.underlyingClient, - savedObjectsClient, - eventLogService, - logger, - }); + agent.setTransactionName(`${options.rule.ruleTypeId} execution`); + return withSecuritySpan('scurityRuleTypeExecutor', async () => { + const { + alertId, + params, + previousStartedAt, + startedAt, + services, + spaceId, + state, + updatedBy: updatedByUser, + rule, + } = options; + let runState = state; + const { from, maxSignals, meta, ruleId, timestampOverride, to } = params; + const { alertWithPersistence, savedObjectsClient, scopedClusterClient } = services; + const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); + + const esClient = scopedClusterClient.asCurrentUser; + + const ruleStatusClient = ruleExecutionLogClientOverride + ? ruleExecutionLogClientOverride + : new RuleExecutionLogClient({ + underlyingClient: config.ruleExecutionLog.underlyingClient, + savedObjectsClient, + eventLogService, + logger, + }); - const completeRule = { - ruleConfig: rule, - ruleParams: params, - alertId, - }; - - const { - actions, - name, - schedule: { interval }, - ruleTypeId, - } = completeRule.ruleConfig; - - const refresh = actions.length ? 'wait_for' : false; - - const buildRuleMessage = buildRuleMessageFactory({ - id: alertId, - ruleId, - name, - index: spaceId, - }); + const completeRule = { + ruleConfig: rule, + ruleParams: params, + alertId, + }; + + const { + actions, + name, + schedule: { interval }, + ruleTypeId, + } = completeRule.ruleConfig; + + const refresh = actions.length ? 'wait_for' : false; + + const buildRuleMessage = buildRuleMessageFactory({ + id: alertId, + ruleId, + name, + index: spaceId, + }); - logger.debug(buildRuleMessage('[+] Starting Signal Rule execution')); - logger.debug(buildRuleMessage(`interval: ${interval}`)); - - let wroteWarningStatus = false; - const basicLogArguments = { - spaceId, - ruleId: alertId, - ruleName: name, - ruleType: ruleTypeId, - }; - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus['going to run'], - }); + logger.debug(buildRuleMessage('[+] Starting Signal Rule execution')); + logger.debug(buildRuleMessage(`interval: ${interval}`)); - let result = createResultObject(state); - - const notificationRuleParams: NotificationRuleTypeParams = { - ...params, - name, - id: alertId, - } as unknown as NotificationRuleTypeParams; - - // check if rule has permissions to access given index pattern - // move this collection of lines into a function in utils - // so that we can use it in create rules route, bulk, etc. - try { - // Typescript 4.1.3 can't figure out that `!isMachineLearningParams(params)` also excludes the only rule type - // of rule params that doesn't include `params.index`, but Typescript 4.3.5 does compute the stricter type correctly. - // When we update Typescript to >= 4.3.5, we can replace this logic with `!isMachineLearningParams(params)` again. - if ( - isEqlParams(params) || - isThresholdParams(params) || - isQueryParams(params) || - isSavedQueryParams(params) || - isThreatParams(params) - ) { - const index = params.index; - const hasTimestampOverride = !!timestampOverride; - - const inputIndices = params.index ?? []; - - const privileges = await checkPrivilegesFromEsClient(esClient, inputIndices); - - wroteWarningStatus = await hasReadIndexPrivileges({ - ...basicLogArguments, - privileges, - logger, - buildRuleMessage, - ruleStatusClient, - }); + let wroteWarningStatus = false; + const basicLogArguments = { + spaceId, + ruleId: alertId, + ruleName: name, + ruleType: ruleTypeId, + }; + await ruleStatusClient.logStatusChange({ + ...basicLogArguments, + newStatus: RuleExecutionStatus['going to run'], + }); - if (!wroteWarningStatus) { - const timestampFieldCaps = await services.scopedClusterClient.asCurrentUser.fieldCaps( - { - index, - fields: hasTimestampOverride - ? ['@timestamp', timestampOverride as string] - : ['@timestamp'], - include_unmapped: true, - } - ); - wroteWarningStatus = await hasTimestampFields({ + let result = createResultObject(state); + + const notificationRuleParams: NotificationRuleTypeParams = { + ...params, + name, + id: alertId, + } as unknown as NotificationRuleTypeParams; + + // check if rule has permissions to access given index pattern + // move this collection of lines into a function in utils + // so that we can use it in create rules route, bulk, etc. + try { + // Typescript 4.1.3 can't figure out that `!isMachineLearningParams(params)` also excludes the only rule type + // of rule params that doesn't include `params.index`, but Typescript 4.3.5 does compute the stricter type correctly. + // When we update Typescript to >= 4.3.5, we can replace this logic with `!isMachineLearningParams(params)` again. + if ( + isEqlParams(params) || + isThresholdParams(params) || + isQueryParams(params) || + isSavedQueryParams(params) || + isThreatParams(params) + ) { + const index = params.index; + const hasTimestampOverride = !!timestampOverride; + + const inputIndices = params.index ?? []; + + const privileges = await checkPrivilegesFromEsClient(esClient, inputIndices); + + wroteWarningStatus = await hasReadIndexPrivileges({ ...basicLogArguments, - timestampField: hasTimestampOverride ? (timestampOverride as string) : '@timestamp', - timestampFieldCapsResponse: timestampFieldCaps, - inputIndices, - ruleStatusClient, + privileges, logger, buildRuleMessage, + ruleStatusClient, }); + + if (!wroteWarningStatus) { + const timestampFieldCaps = await withSecuritySpan('fieldCaps', () => + services.scopedClusterClient.asCurrentUser.fieldCaps({ + index, + fields: hasTimestampOverride + ? ['@timestamp', timestampOverride as string] + : ['@timestamp'], + include_unmapped: true, + }) + ); + wroteWarningStatus = await hasTimestampFields({ + ...basicLogArguments, + timestampField: hasTimestampOverride + ? (timestampOverride as string) + : '@timestamp', + timestampFieldCapsResponse: timestampFieldCaps, + inputIndices, + ruleStatusClient, + logger, + buildRuleMessage, + }); + } } + } catch (exc) { + const errorMessage = buildRuleMessage(`Check privileges failed to execute ${exc}`); + logger.error(errorMessage); + await ruleStatusClient.logStatusChange({ + ...basicLogArguments, + message: errorMessage, + newStatus: RuleExecutionStatus['partial failure'], + }); + wroteWarningStatus = true; } - } catch (exc) { - const errorMessage = buildRuleMessage(`Check privileges failed to execute ${exc}`); - logger.error(errorMessage); - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - message: errorMessage, - newStatus: RuleExecutionStatus['partial failure'], - }); - wroteWarningStatus = true; - } - let hasError = false; - const { tuples, remainingGap } = getRuleRangeTuples({ - logger, - previousStartedAt, - from: from as string, - to: to as string, - interval, - maxSignals: maxSignals ?? DEFAULT_MAX_SIGNALS, - buildRuleMessage, - startedAt, - }); - - if (remainingGap.asMilliseconds() > 0) { - const gapString = remainingGap.humanize(); - const gapMessage = buildRuleMessage( - `${gapString} (${remainingGap.asMilliseconds()}ms) were not queried between this rule execution and the last execution, so signals may have been missed.`, - 'Consider increasing your look behind time or adding more Kibana instances.' - ); - logger.warn(gapMessage); - hasError = true; - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus.failed, - message: gapMessage, - metrics: { executionGap: remainingGap }, - }); - } - - try { - const { listClient, exceptionsClient } = getListClient({ - esClient: services.scopedClusterClient.asCurrentUser, - updatedByUser, - spaceId, - lists, - savedObjectClient: options.services.savedObjectsClient, - }); - - const exceptionItems = await getExceptions({ - client: exceptionsClient, - lists: (params.exceptionsList as ListArray) ?? [], - }); - - const bulkCreate = bulkCreateFactory( + let hasError = false; + const { tuples, remainingGap } = getRuleRangeTuples({ logger, - alertWithPersistence, + previousStartedAt, + from: from as string, + to: to as string, + interval, + maxSignals: maxSignals ?? DEFAULT_MAX_SIGNALS, buildRuleMessage, - refresh - ); - - const legacySignalFields: string[] = Object.keys(aadFieldConversion); - const wrapHits = wrapHitsFactory({ - ignoreFields: [...ignoreFields, ...legacySignalFields], - mergeStrategy, - completeRule, - spaceId, + startedAt, }); - const wrapSequences = wrapSequencesFactory({ - logger, - ignoreFields: [...ignoreFields, ...legacySignalFields], - mergeStrategy, - completeRule, - spaceId, - }); + if (remainingGap.asMilliseconds() > 0) { + const gapString = remainingGap.humanize(); + const gapMessage = buildRuleMessage( + `${gapString} (${remainingGap.asMilliseconds()}ms) were not queried between this rule execution and the last execution, so signals may have been missed.`, + 'Consider increasing your look behind time or adding more Kibana instances.' + ); + logger.warn(gapMessage); + hasError = true; + await ruleStatusClient.logStatusChange({ + ...basicLogArguments, + newStatus: RuleExecutionStatus.failed, + message: gapMessage, + metrics: { executionGap: remainingGap }, + }); + } - for (const tuple of tuples) { - const runResult = await type.executor({ - ...options, - services, - state: runState, - runOpts: { - buildRuleMessage, - bulkCreate, - exceptionItems, - listClient, - completeRule, - searchAfterSize, - tuple, - wrapHits, - wrapSequences, - }, + try { + const { listClient, exceptionsClient } = getListClient({ + esClient: services.scopedClusterClient.asCurrentUser, + updatedByUser, + spaceId, + lists, + savedObjectClient: options.services.savedObjectsClient, }); - const createdSignals = result.createdSignals.concat(runResult.createdSignals); - const warningMessages = result.warningMessages.concat(runResult.warningMessages); - result = { - bulkCreateTimes: result.bulkCreateTimes.concat(runResult.bulkCreateTimes), - createdSignals, - createdSignalsCount: createdSignals.length, - errors: result.errors.concat(runResult.errors), - lastLookbackDate: runResult.lastLookBackDate, - searchAfterTimes: result.searchAfterTimes.concat(runResult.searchAfterTimes), - state: runState, - success: result.success && runResult.success, - warning: warningMessages.length > 0, - warningMessages, - }; - runState = runResult.state; - } + const exceptionItems = await getExceptions({ + client: exceptionsClient, + lists: (params.exceptionsList as ListArray) ?? [], + }); - if (result.warningMessages.length) { - const warningMessage = buildRuleMessage( - truncateMessageList(result.warningMessages).join() + const bulkCreate = bulkCreateFactory( + logger, + alertWithPersistence, + buildRuleMessage, + refresh ); - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus['partial failure'], - message: warningMessage, + + const legacySignalFields: string[] = Object.keys(aadFieldConversion); + const wrapHits = wrapHitsFactory({ + ignoreFields: [...ignoreFields, ...legacySignalFields], + mergeStrategy, + completeRule, + spaceId, }); - } - if (result.success) { - const createdSignalsCount = result.createdSignals.length; + const wrapSequences = wrapSequencesFactory({ + logger, + ignoreFields: [...ignoreFields, ...legacySignalFields], + mergeStrategy, + completeRule, + spaceId, + }); - if (actions.length) { - const fromInMs = parseScheduleDates(`now-${interval}`)?.format('x'); - const toInMs = parseScheduleDates('now')?.format('x'); - const resultsLink = getNotificationResultsLink({ - from: fromInMs, - to: toInMs, - id: alertId, - kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) - ?.kibana_siem_app_url, + for (const tuple of tuples) { + const runResult = await type.executor({ + ...options, + services, + state: runState, + runOpts: { + buildRuleMessage, + bulkCreate, + exceptionItems, + listClient, + completeRule, + searchAfterSize, + tuple, + wrapHits, + wrapSequences, + }, }); + const createdSignals = result.createdSignals.concat(runResult.createdSignals); + const warningMessages = result.warningMessages.concat(runResult.warningMessages); + result = { + bulkCreateTimes: result.bulkCreateTimes.concat(runResult.bulkCreateTimes), + createdSignals, + createdSignalsCount: createdSignals.length, + errors: result.errors.concat(runResult.errors), + lastLookbackDate: runResult.lastLookBackDate, + searchAfterTimes: result.searchAfterTimes.concat(runResult.searchAfterTimes), + state: runState, + success: result.success && runResult.success, + warning: warningMessages.length > 0, + warningMessages, + }; + runState = runResult.state; + } + + if (result.warningMessages.length) { + const warningMessage = buildRuleMessage( + truncateMessageList(result.warningMessages).join() + ); + await ruleStatusClient.logStatusChange({ + ...basicLogArguments, + newStatus: RuleExecutionStatus['partial failure'], + message: warningMessage, + }); + } + + if (result.success) { + const createdSignalsCount = result.createdSignals.length; + + if (actions.length) { + const fromInMs = parseScheduleDates(`now-${interval}`)?.format('x'); + const toInMs = parseScheduleDates('now')?.format('x'); + const resultsLink = getNotificationResultsLink({ + from: fromInMs, + to: toInMs, + id: alertId, + kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) + ?.kibana_siem_app_url, + }); + + logger.debug( + buildRuleMessage(`Found ${createdSignalsCount} signals for notification.`) + ); + + if (completeRule.ruleConfig.throttle != null) { + await scheduleThrottledNotificationActions({ + alertInstance: services.alertInstanceFactory(alertId), + throttle: completeRule.ruleConfig.throttle ?? '', + startedAt, + id: alertId, + kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) + ?.kibana_siem_app_url, + outputIndex: ruleDataClient.indexName, + ruleId, + esClient: services.scopedClusterClient.asCurrentUser, + notificationRuleParams, + signals: result.createdSignals, + logger, + }); + } else if (createdSignalsCount) { + const alertInstance = services.alertInstanceFactory(alertId); + scheduleNotificationActions({ + alertInstance, + signalsCount: createdSignalsCount, + signals: result.createdSignals, + resultsLink, + ruleParams: notificationRuleParams, + }); + } + } + + logger.debug(buildRuleMessage('[+] Signal Rule execution completed.')); logger.debug( - buildRuleMessage(`Found ${createdSignalsCount} signals for notification.`) + buildRuleMessage( + `[+] Finished indexing ${createdSignalsCount} signals into ${ruleDataClient.indexName}` + ) ); + if (!hasError && !wroteWarningStatus && !result.warning) { + await ruleStatusClient.logStatusChange({ + ...basicLogArguments, + newStatus: RuleExecutionStatus.succeeded, + message: 'succeeded', + metrics: { + indexingDurations: result.bulkCreateTimes, + searchDurations: result.searchAfterTimes, + lastLookBackDate: result.lastLookbackDate?.toISOString(), + }, + }); + } + + logger.debug( + buildRuleMessage( + `[+] Finished indexing ${createdSignalsCount} ${ + !isEmpty(tuples) + ? `signals searched between date ranges ${JSON.stringify(tuples, null, 2)}` + : '' + }` + ) + ); + } else { + // NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early if (completeRule.ruleConfig.throttle != null) { await scheduleThrottledNotificationActions({ alertInstance: services.alertInstanceFactory(alertId), throttle: completeRule.ruleConfig.throttle ?? '', startedAt, - id: alertId, + id: completeRule.alertId, kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) ?.kibana_siem_app_url, outputIndex: ruleDataClient.indexName, @@ -330,30 +394,16 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = signals: result.createdSignals, logger, }); - } else if (createdSignalsCount) { - const alertInstance = services.alertInstanceFactory(alertId); - scheduleNotificationActions({ - alertInstance, - signalsCount: createdSignalsCount, - signals: result.createdSignals, - resultsLink, - ruleParams: notificationRuleParams, - }); } - } - - logger.debug(buildRuleMessage('[+] Signal Rule execution completed.')); - logger.debug( - buildRuleMessage( - `[+] Finished indexing ${createdSignalsCount} signals into ${ruleDataClient.indexName}` - ) - ); - - if (!hasError && !wroteWarningStatus && !result.warning) { + const errorMessage = buildRuleMessage( + 'Bulk Indexing of signals failed:', + truncateMessageList(result.errors).join() + ); + logger.error(errorMessage); await ruleStatusClient.logStatusChange({ ...basicLogArguments, - newStatus: RuleExecutionStatus.succeeded, - message: 'succeeded', + newStatus: RuleExecutionStatus.failed, + message: errorMessage, metrics: { indexingDurations: result.bulkCreateTimes, searchDurations: result.searchAfterTimes, @@ -361,17 +411,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = }, }); } - - logger.debug( - buildRuleMessage( - `[+] Finished indexing ${createdSignalsCount} ${ - !isEmpty(tuples) - ? `signals searched between date ranges ${JSON.stringify(tuples, null, 2)}` - : '' - }` - ) - ); - } else { + } catch (error) { // NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early if (completeRule.ruleConfig.throttle != null) { await scheduleThrottledNotificationActions({ @@ -389,15 +429,18 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = logger, }); } - const errorMessage = buildRuleMessage( - 'Bulk Indexing of signals failed:', - truncateMessageList(result.errors).join() + + const errorMessage = error.message ?? '(no error message given)'; + const message = buildRuleMessage( + 'An error occurred during rule execution:', + `message: "${errorMessage}"` ); - logger.error(errorMessage); + + logger.error(message); await ruleStatusClient.logStatusChange({ ...basicLogArguments, newStatus: RuleExecutionStatus.failed, - message: errorMessage, + message, metrics: { indexingDurations: result.bulkCreateTimes, searchDurations: result.searchAfterTimes, @@ -405,45 +448,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = }, }); } - } catch (error) { - // NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early - if (completeRule.ruleConfig.throttle != null) { - await scheduleThrottledNotificationActions({ - alertInstance: services.alertInstanceFactory(alertId), - throttle: completeRule.ruleConfig.throttle ?? '', - startedAt, - id: completeRule.alertId, - kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) - ?.kibana_siem_app_url, - outputIndex: ruleDataClient.indexName, - ruleId, - esClient: services.scopedClusterClient.asCurrentUser, - notificationRuleParams, - signals: result.createdSignals, - logger, - }); - } - - const errorMessage = error.message ?? '(no error message given)'; - const message = buildRuleMessage( - 'An error occurred during rule execution:', - `message: "${errorMessage}"` - ); - logger.error(message); - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus.failed, - message, - metrics: { - indexingDurations: result.bulkCreateTimes, - searchDurations: result.searchAfterTimes, - lastLookBackDate: result.lastLookbackDate?.toISOString(), - }, - }); - } - - return result.state; + return result.state; + }); }, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts index 0d08008be72bc..4bfb6149a1d29 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts @@ -13,6 +13,7 @@ import { BuildRuleMessage } from './rule_messages'; import { RefreshTypes } from '../types'; import { BaseHit } from '../../../../common/detection_engine/types'; import { errorAggregator, makeFloatString } from './utils'; +import { withSecuritySpan } from '../../../utils/with_security_span'; export interface GenericBulkCreateResponse { success: boolean; @@ -52,10 +53,12 @@ export const bulkCreateFactory = ]); const start = performance.now(); - const { body: response } = await esClient.bulk({ - refresh: refreshForBulkCreate, - body: bulkBody, - }); + const { body: response } = await withSecuritySpan('writeAlertsBulk', () => + esClient.bulk({ + refresh: refreshForBulkCreate, + body: bulkBody, + }) + ); const end = performance.now(); logger.debug( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index 61a8fb930efed..7e83649d3e38b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -34,6 +34,7 @@ import { createSearchAfterReturnType, makeFloatString } from '../utils'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; import { buildReasonMessageForEqlAlert } from '../reason_formatters'; import { CompleteRule, EqlRuleParams } from '../../schemas/rule_schemas'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; export const eqlExecutor = async ({ completeRule, @@ -60,89 +61,90 @@ export const eqlExecutor = async ({ wrapHits: WrapHits; wrapSequences: WrapSequences; }): Promise => { - const result = createSearchAfterReturnType(); - const ruleParams = completeRule.ruleParams; - if (hasLargeValueItem(exceptionItems)) { - result.warningMessages.push( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' - ); - result.warning = true; - } - if (!experimentalFeatures.ruleRegistryEnabled) { - try { - const signalIndexVersion = await getIndexVersion( - services.scopedClusterClient.asCurrentUser, - ruleParams.outputIndex + return withSecuritySpan('eqlExecutor', async () => { + const result = createSearchAfterReturnType(); + if (hasLargeValueItem(exceptionItems)) { + result.warningMessages.push( + 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' ); - if (isOutdated({ current: signalIndexVersion, target: MIN_EQL_RULE_INDEX_VERSION })) { - throw new Error( - `EQL based rules require an update to version ${MIN_EQL_RULE_INDEX_VERSION} of the detection alerts index mapping` - ); - } - } catch (err) { - if (err.statusCode === 403) { - throw new Error( - `EQL based rules require the user that created it to have the view_index_metadata, read, and write permissions for index: ${ruleParams.outputIndex}` + result.warning = true; + } + if (!experimentalFeatures.ruleRegistryEnabled) { + try { + const signalIndexVersion = await getIndexVersion( + services.scopedClusterClient.asCurrentUser, + ruleParams.outputIndex ); - } else { - throw err; + if (isOutdated({ current: signalIndexVersion, target: MIN_EQL_RULE_INDEX_VERSION })) { + throw new Error( + `EQL based rules require an update to version ${MIN_EQL_RULE_INDEX_VERSION} of the detection alerts index mapping` + ); + } + } catch (err) { + if (err.statusCode === 403) { + throw new Error( + `EQL based rules require the user that created it to have the view_index_metadata, read, and write permissions for index: ${ruleParams.outputIndex}` + ); + } else { + throw err; + } } } - } - const inputIndex = await getInputIndex({ - experimentalFeatures, - services, - version, - index: ruleParams.index, - }); + const inputIndex = await getInputIndex({ + experimentalFeatures, + services, + version, + index: ruleParams.index, + }); - const request = buildEqlSearchRequest( - ruleParams.query, - inputIndex, - tuple.from.toISOString(), - tuple.to.toISOString(), - searchAfterSize, - ruleParams.timestampOverride, - exceptionItems, - ruleParams.eventCategoryOverride - ); + const request = buildEqlSearchRequest( + ruleParams.query, + inputIndex, + tuple.from.toISOString(), + tuple.to.toISOString(), + searchAfterSize, + ruleParams.timestampOverride, + exceptionItems, + ruleParams.eventCategoryOverride + ); - const eqlSignalSearchStart = performance.now(); - logger.debug( - `EQL query request path: ${request.path}, method: ${request.method}, body: ${JSON.stringify( - request.body - )}` - ); + const eqlSignalSearchStart = performance.now(); + logger.debug( + `EQL query request path: ${request.path}, method: ${request.method}, body: ${JSON.stringify( + request.body + )}` + ); - // TODO: fix this later - const { body: response } = (await services.scopedClusterClient.asCurrentUser.transport.request( - request - )) as TransportResult; + // TODO: fix this later + const { body: response } = (await services.scopedClusterClient.asCurrentUser.transport.request( + request + )) as TransportResult; - const eqlSignalSearchEnd = performance.now(); - const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart); - result.searchAfterTimes = [eqlSearchDuration]; + const eqlSignalSearchEnd = performance.now(); + const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart); + result.searchAfterTimes = [eqlSearchDuration]; - let newSignals: SimpleHit[] | undefined; - if (response.hits.sequences !== undefined) { - newSignals = wrapSequences(response.hits.sequences, buildReasonMessageForEqlAlert); - } else if (response.hits.events !== undefined) { - newSignals = wrapHits(response.hits.events, buildReasonMessageForEqlAlert); - } else { - throw new Error( - 'eql query response should have either `sequences` or `events` but had neither' - ); - } + let newSignals: SimpleHit[] | undefined; + if (response.hits.sequences !== undefined) { + newSignals = wrapSequences(response.hits.sequences, buildReasonMessageForEqlAlert); + } else if (response.hits.events !== undefined) { + newSignals = wrapHits(response.hits.events, buildReasonMessageForEqlAlert); + } else { + throw new Error( + 'eql query response should have either `sequences` or `events` but had neither' + ); + } - if (newSignals?.length) { - const insertResult = await bulkCreate(newSignals); - result.bulkCreateTimes.push(insertResult.bulkCreateDuration); - result.createdSignalsCount += insertResult.createdItemsCount; - result.createdSignals = insertResult.createdItems; - } + if (newSignals?.length) { + const insertResult = await bulkCreate(newSignals); + result.bulkCreateTimes.push(insertResult.bulkCreateDuration); + result.createdSignalsCount += insertResult.createdItemsCount; + result.createdSignals = insertResult.createdItems; + } - result.success = true; - return result; + result.success = true; + return result; + }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts index 3db8d51ab76ed..3610c45017019 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts @@ -22,6 +22,7 @@ import { BuildRuleMessage } from '../rule_messages'; import { BulkCreate, RuleRangeTuple, WrapHits } from '../types'; import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils'; import { SetupPlugins } from '../../../../plugin'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; export const mlExecutor = async ({ completeRule, @@ -48,94 +49,97 @@ export const mlExecutor = async ({ }) => { const result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; - if (ml == null) { - throw new Error('ML plugin unavailable during rule execution'); - } - // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is - // currently unused by the jobsSummary function. - const fakeRequest = {} as KibanaRequest; - const summaryJobs = await ml - .jobServiceProvider(fakeRequest, services.savedObjectsClient) - .jobsSummary(ruleParams.machineLearningJobId); - const jobSummaries = summaryJobs.filter((job) => - ruleParams.machineLearningJobId.includes(job.id) - ); + return withSecuritySpan('mlExecutor', async () => { + if (ml == null) { + throw new Error('ML plugin unavailable during rule execution'); + } - if ( - jobSummaries.length < 1 || - jobSummaries.some((job) => !isJobStarted(job.jobState, job.datafeedState)) - ) { - const warningMessage = buildRuleMessage( - 'Machine learning job(s) are not started:', - ...jobSummaries.map((job) => - [ - `job id: "${job.id}"`, - `job status: "${job.jobState}"`, - `datafeed status: "${job.datafeedState}"`, - ].join(', ') - ) + // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is + // currently unused by the jobsSummary function. + const fakeRequest = {} as KibanaRequest; + const summaryJobs = await ml + .jobServiceProvider(fakeRequest, services.savedObjectsClient) + .jobsSummary(ruleParams.machineLearningJobId); + const jobSummaries = summaryJobs.filter((job) => + ruleParams.machineLearningJobId.includes(job.id) ); - result.warningMessages.push(warningMessage); - logger.warn(warningMessage); - result.warning = true; - } - const anomalyResults = await findMlSignals({ - ml, - // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is - // currently unused by the mlAnomalySearch function. - request: {} as unknown as KibanaRequest, - savedObjectsClient: services.savedObjectsClient, - jobIds: ruleParams.machineLearningJobId, - anomalyThreshold: ruleParams.anomalyThreshold, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - exceptionItems, - }); + if ( + jobSummaries.length < 1 || + jobSummaries.some((job) => !isJobStarted(job.jobState, job.datafeedState)) + ) { + const warningMessage = buildRuleMessage( + 'Machine learning job(s) are not started:', + ...jobSummaries.map((job) => + [ + `job id: "${job.id}"`, + `job status: "${job.jobState}"`, + `datafeed status: "${job.datafeedState}"`, + ].join(', ') + ) + ); + result.warningMessages.push(warningMessage); + logger.warn(warningMessage); + result.warning = true; + } - const filteredAnomalyResults = await filterEventsAgainstList({ - listClient, - exceptionsList: exceptionItems, - logger, - eventSearchResult: anomalyResults, - buildRuleMessage, - }); + const anomalyResults = await findMlSignals({ + ml, + // Using fake KibanaRequest as it is needed to satisfy the ML Services API, but can be empty as it is + // currently unused by the mlAnomalySearch function. + request: {} as unknown as KibanaRequest, + savedObjectsClient: services.savedObjectsClient, + jobIds: ruleParams.machineLearningJobId, + anomalyThreshold: ruleParams.anomalyThreshold, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + exceptionItems, + }); - const anomalyCount = filteredAnomalyResults.hits.hits.length; - if (anomalyCount) { - logger.debug(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`)); - } - const { success, errors, bulkCreateDuration, createdItemsCount, createdItems } = - await bulkCreateMlSignals({ - someResult: filteredAnomalyResults, - completeRule, - services, + const filteredAnomalyResults = await filterEventsAgainstList({ + listClient, + exceptionsList: exceptionItems, logger, - id: completeRule.alertId, - signalsIndex: ruleParams.outputIndex, + eventSearchResult: anomalyResults, buildRuleMessage, - bulkCreate, - wrapHits, }); - // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } - const shardFailures = - ( - filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & { - failures: []; - } - ).failures ?? []; - const searchErrors = createErrorsFromShard({ - errors: shardFailures, + + const anomalyCount = filteredAnomalyResults.hits.hits.length; + if (anomalyCount) { + logger.debug(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`)); + } + const { success, errors, bulkCreateDuration, createdItemsCount, createdItems } = + await bulkCreateMlSignals({ + someResult: filteredAnomalyResults, + completeRule, + services, + logger, + id: completeRule.alertId, + signalsIndex: ruleParams.outputIndex, + buildRuleMessage, + bulkCreate, + wrapHits, + }); + // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } + const shardFailures = + ( + filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & { + failures: []; + } + ).failures ?? []; + const searchErrors = createErrorsFromShard({ + errors: shardFailures, + }); + return mergeReturns([ + result, + createSearchAfterReturnType({ + success: success && filteredAnomalyResults._shards.failed === 0, + errors: [...errors, ...searchErrors], + createdSignalsCount: createdItemsCount, + createdSignals: createdItems, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + }), + ]); }); - return mergeReturns([ - result, - createSearchAfterReturnType({ - success: success && filteredAnomalyResults._shards.failed === 0, - errors: [...errors, ...searchErrors], - createdSignalsCount: createdItemsCount, - createdSignals: createdItems, - bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], - }), - ]); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts index 2bee175f357f3..47492f1db7fa9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -22,6 +22,7 @@ import { BuildRuleMessage } from '../rule_messages'; import { CompleteRule, SavedQueryRuleParams, QueryRuleParams } from '../../schemas/rule_schemas'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; import { buildReasonMessageForQueryAlert } from '../reason_formatters'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; export const queryExecutor = async ({ completeRule, @@ -54,40 +55,42 @@ export const queryExecutor = async ({ }) => { const ruleParams = completeRule.ruleParams; - const inputIndex = await getInputIndex({ - experimentalFeatures, - services, - version, - index: ruleParams.index, - }); + return withSecuritySpan('queryExecutor', async () => { + const inputIndex = await getInputIndex({ + experimentalFeatures, + services, + version, + index: ruleParams.index, + }); - const esFilter = await getFilter({ - type: ruleParams.type, - filters: ruleParams.filters, - language: ruleParams.language, - query: ruleParams.query, - savedId: ruleParams.savedId, - services, - index: inputIndex, - lists: exceptionItems, - }); + const esFilter = await getFilter({ + type: ruleParams.type, + filters: ruleParams.filters, + language: ruleParams.language, + query: ruleParams.query, + savedId: ruleParams.savedId, + services, + index: inputIndex, + lists: exceptionItems, + }); - return searchAfterAndBulkCreate({ - tuple, - listClient, - exceptionsList: exceptionItems, - completeRule, - services, - logger, - eventsTelemetry, - id: completeRule.alertId, - inputIndexPattern: inputIndex, - signalsIndex: ruleParams.outputIndex, - filter: esFilter, - pageSize: searchAfterSize, - buildReasonMessage: buildReasonMessageForQueryAlert, - buildRuleMessage, - bulkCreate, - wrapHits, + return searchAfterAndBulkCreate({ + tuple, + listClient, + exceptionsList: exceptionItems, + completeRule, + services, + logger, + eventsTelemetry, + id: completeRule.alertId, + inputIndexPattern: inputIndex, + signalsIndex: ruleParams.outputIndex, + filter: esFilter, + pageSize: searchAfterSize, + buildReasonMessage: buildReasonMessageForQueryAlert, + buildRuleMessage, + bulkCreate, + wrapHits, + }); }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts index f2e2590ac1e2d..43440444a6537 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts @@ -20,6 +20,7 @@ import { BuildRuleMessage } from '../rule_messages'; import { createThreatSignals } from '../threat_mapping/create_threat_signals'; import { CompleteRule, ThreatRuleParams } from '../../schemas/rule_schemas'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; export const threatMatchExecutor = async ({ completeRule, @@ -51,39 +52,42 @@ export const threatMatchExecutor = async ({ wrapHits: WrapHits; }) => { const ruleParams = completeRule.ruleParams; - const inputIndex = await getInputIndex({ - experimentalFeatures, - services, - version, - index: ruleParams.index, - }); - return createThreatSignals({ - alertId: completeRule.alertId, - buildRuleMessage, - bulkCreate, - completeRule, - concurrentSearches: ruleParams.concurrentSearches ?? 1, - eventsTelemetry, - exceptionItems, - filters: ruleParams.filters ?? [], - inputIndex, - itemsPerSearch: ruleParams.itemsPerSearch ?? 9000, - language: ruleParams.language, - listClient, - logger, - outputIndex: ruleParams.outputIndex, - query: ruleParams.query, - savedId: ruleParams.savedId, - searchAfterSize, - services, - threatFilters: ruleParams.threatFilters ?? [], - threatIndex: ruleParams.threatIndex, - threatIndicatorPath: ruleParams.threatIndicatorPath, - threatLanguage: ruleParams.threatLanguage, - threatMapping: ruleParams.threatMapping, - threatQuery: ruleParams.threatQuery, - tuple, - type: ruleParams.type, - wrapHits, + + return withSecuritySpan('threatMatchExecutor', async () => { + const inputIndex = await getInputIndex({ + experimentalFeatures, + services, + version, + index: ruleParams.index, + }); + return createThreatSignals({ + alertId: completeRule.alertId, + buildRuleMessage, + bulkCreate, + completeRule, + concurrentSearches: ruleParams.concurrentSearches ?? 1, + eventsTelemetry, + exceptionItems, + filters: ruleParams.filters ?? [], + inputIndex, + itemsPerSearch: ruleParams.itemsPerSearch ?? 9000, + language: ruleParams.language, + listClient, + logger, + outputIndex: ruleParams.outputIndex, + query: ruleParams.query, + savedId: ruleParams.savedId, + searchAfterSize, + services, + threatFilters: ruleParams.threatFilters ?? [], + threatIndex: ruleParams.threatIndex, + threatIndicatorPath: ruleParams.threatIndicatorPath, + threatLanguage: ruleParams.threatLanguage, + threatMapping: ruleParams.threatMapping, + threatQuery: ruleParams.threatQuery, + tuple, + type: ruleParams.type, + wrapHits, + }); }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index 2bb5d6880c634..34e6e26a30eab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -39,6 +39,7 @@ import { } from '../utils'; import { BuildRuleMessage } from '../rule_messages'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; import { buildThresholdSignalHistory } from '../threshold/build_signal_history'; export const thresholdExecutor = async ({ @@ -71,136 +72,138 @@ export const thresholdExecutor = async ({ let result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; - // Get state or build initial state (on upgrade) - const { signalHistory, searchErrors: previousSearchErrors } = state.initialized - ? { signalHistory: state.signalHistory, searchErrors: [] } - : await getThresholdSignalHistory({ - indexPattern: ['*'], // TODO: get outputIndex? - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - ruleId: ruleParams.ruleId, - bucketByFields: ruleParams.threshold.field, - timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); - - if (!state.initialized) { - // Clean up any signal history that has fallen outside the window - const toDelete: string[] = []; - for (const [hash, entry] of Object.entries(signalHistory)) { - if (entry.lastSignalTimestamp < tuple.from.valueOf()) { - toDelete.push(hash); + return withSecuritySpan('thresholdExecutor', async () => { + // Get state or build initial state (on upgrade) + const { signalHistory, searchErrors: previousSearchErrors } = state.initialized + ? { signalHistory: state.signalHistory, searchErrors: [] } + : await getThresholdSignalHistory({ + indexPattern: ['*'], // TODO: get outputIndex? + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + ruleId: ruleParams.ruleId, + bucketByFields: ruleParams.threshold.field, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); + + if (!state.initialized) { + // Clean up any signal history that has fallen outside the window + const toDelete: string[] = []; + for (const [hash, entry] of Object.entries(signalHistory)) { + if (entry.lastSignalTimestamp < tuple.from.valueOf()) { + toDelete.push(hash); + } + } + for (const hash of toDelete) { + delete signalHistory[hash]; } } - for (const hash of toDelete) { - delete signalHistory[hash]; - } - } - - if (hasLargeValueItem(exceptionItems)) { - result.warningMessages.push( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' - ); - result.warning = true; - } - - const inputIndex = await getInputIndex({ - experimentalFeatures, - services, - version, - index: ruleParams.index, - }); - const bucketFilters = await getThresholdBucketFilters({ - signalHistory, - timestampOverride: ruleParams.timestampOverride, - }); + if (hasLargeValueItem(exceptionItems)) { + result.warningMessages.push( + 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' + ); + result.warning = true; + } - const esFilter = await getFilter({ - type: ruleParams.type, - filters: ruleParams.filters ? ruleParams.filters.concat(bucketFilters) : bucketFilters, - language: ruleParams.language, - query: ruleParams.query, - savedId: ruleParams.savedId, - services, - index: inputIndex, - lists: exceptionItems, - }); + const inputIndex = await getInputIndex({ + experimentalFeatures, + services, + version, + index: ruleParams.index, + }); - const { - searchResult: thresholdResults, - searchErrors, - searchDuration: thresholdSearchDuration, - } = await findThresholdSignals({ - inputIndexPattern: inputIndex, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - filter: esFilter, - threshold: ruleParams.threshold, - timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); + const bucketFilters = await getThresholdBucketFilters({ + signalHistory, + timestampOverride: ruleParams.timestampOverride, + }); - const { success, bulkCreateDuration, createdItemsCount, createdItems, errors } = - await bulkCreateThresholdSignals({ - someResult: thresholdResults, - completeRule, - filter: esFilter, + const esFilter = await getFilter({ + type: ruleParams.type, + filters: ruleParams.filters ? ruleParams.filters.concat(bucketFilters) : bucketFilters, + language: ruleParams.language, + query: ruleParams.query, + savedId: ruleParams.savedId, services, - logger, - inputIndexPattern: inputIndex, - signalsIndex: ruleParams.outputIndex, - startedAt, - from: tuple.from.toDate(), - signalHistory, - bulkCreate, - wrapHits, + index: inputIndex, + lists: exceptionItems, }); - result = mergeReturns([ - result, - createSearchAfterReturnTypeFromResponse({ + const { searchResult: thresholdResults, + searchErrors, + searchDuration: thresholdSearchDuration, + } = await findThresholdSignals({ + inputIndexPattern: inputIndex, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + filter: esFilter, + threshold: ruleParams.threshold, timestampOverride: ruleParams.timestampOverride, - }), - createSearchAfterReturnType({ - success, - errors: [...errors, ...previousSearchErrors, ...searchErrors], - createdSignalsCount: createdItemsCount, - createdSignals: createdItems, - bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], - searchAfterTimes: [thresholdSearchDuration], - }), - ]); - - const createdAlerts = createdItems.map((alert) => { - const { _id, _index, ...source } = alert as { _id: string; _index: string }; - return { - _id, - _index, - _source: { - ...source, - }, - } as SearchHit; - }); + buildRuleMessage, + }); - const newSignalHistory = buildThresholdSignalHistory({ - alerts: createdAlerts, - }); + const { success, bulkCreateDuration, createdItemsCount, createdItems, errors } = + await bulkCreateThresholdSignals({ + someResult: thresholdResults, + completeRule, + filter: esFilter, + services, + logger, + inputIndexPattern: inputIndex, + signalsIndex: ruleParams.outputIndex, + startedAt, + from: tuple.from.toDate(), + signalHistory, + bulkCreate, + wrapHits, + }); + + result = mergeReturns([ + result, + createSearchAfterReturnTypeFromResponse({ + searchResult: thresholdResults, + timestampOverride: ruleParams.timestampOverride, + }), + createSearchAfterReturnType({ + success, + errors: [...errors, ...previousSearchErrors, ...searchErrors], + createdSignalsCount: createdItemsCount, + createdSignals: createdItems, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + searchAfterTimes: [thresholdSearchDuration], + }), + ]); + + const createdAlerts = createdItems.map((alert) => { + const { _id, _index, ...source } = alert as { _id: string; _index: string }; + return { + _id, + _index, + _source: { + ...source, + }, + } as SearchHit; + }); + + const newSignalHistory = buildThresholdSignalHistory({ + alerts: createdAlerts, + }); - return { - ...result, - state: { - ...state, - initialized: true, - signalHistory: { - ...signalHistory, - ...newSignalHistory, + return { + ...result, + state: { + ...state, + initialized: true, + signalHistory: { + ...signalHistory, + ...newSignalHistory, + }, }, - }, - }; + }; + }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index 574020af45c15..f849900ec75e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -22,6 +22,7 @@ import { } from '../../../../../alerting/server'; import { PartialFilter } from '../types'; import { QueryFilter } from './types'; +import { withSecuritySpan } from '../../../utils/with_security_span'; interface GetFilterArgs { type: Type; @@ -65,9 +66,8 @@ export const getFilter = async ({ if (savedId != null && index != null) { try { // try to get the saved object first - const savedObject = await services.savedObjectsClient.get( - 'query', - savedId + const savedObject = await withSecuritySpan('getSavedFilter', () => + services.savedObjectsClient.get('query', savedId) ); return getQueryFilter( savedObject.attributes.query.query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_input_output_index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_input_output_index.ts index d3b60f1e9a281..3b989004b4cf8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_input_output_index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_input_output_index.ts @@ -16,6 +16,7 @@ import { AlertServices, } from '../../../../../alerting/server'; import { ExperimentalFeatures } from '../../../../common/experimental_features'; +import { withSecuritySpan } from '../../../utils/with_security_span'; export interface GetInputIndex { experimentalFeatures: ExperimentalFeatures; @@ -33,9 +34,11 @@ export const getInputIndex = async ({ if (index != null) { return index; } else { - const configuration = await services.savedObjectsClient.get<{ - 'securitySolution:defaultIndex': string[]; - }>('config', version); + const configuration = await withSecuritySpan('getDefaultIndex', () => + services.savedObjectsClient.get<{ + 'securitySolution:defaultIndex': string[]; + }>('config', version) + ); if (configuration.attributes != null && configuration.attributes[DEFAULT_INDEX_KEY] != null) { return configuration.attributes[DEFAULT_INDEX_KEY]; } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index de8657f73fa55..5469b8c1bb318 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -20,6 +20,7 @@ import { getSafeSortIds, } from './utils'; import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types'; +import { withSecuritySpan } from '../../../utils/with_security_span'; // search_after through documents and re-index using bulk endpoint. export const searchAfterAndBulkCreate = async ({ @@ -41,157 +42,159 @@ export const searchAfterAndBulkCreate = async ({ tuple, wrapHits, }: SearchAfterAndBulkCreateParams): Promise => { - const ruleParams = completeRule.ruleParams; - let toReturn = createSearchAfterReturnType(); + return withSecuritySpan('searchAfterAndBulkCreate', async () => { + const ruleParams = completeRule.ruleParams; + let toReturn = createSearchAfterReturnType(); - // sortId tells us where to start our next consecutive search_after query - let sortIds: estypes.SearchSortResults | undefined; - let hasSortId = true; // default to true so we execute the search on initial run + // sortId tells us where to start our next consecutive search_after query + let sortIds: estypes.SearchSortResults | undefined; + let hasSortId = true; // default to true so we execute the search on initial run - // signalsCreatedCount keeps track of how many signals we have created, - // to ensure we don't exceed maxSignals - let signalsCreatedCount = 0; + // signalsCreatedCount keeps track of how many signals we have created, + // to ensure we don't exceed maxSignals + let signalsCreatedCount = 0; - if (tuple == null || tuple.to == null || tuple.from == null) { - logger.error(buildRuleMessage(`[-] malformed date tuple`)); - return createSearchAfterReturnType({ - success: false, - errors: ['malformed date tuple'], - }); - } - signalsCreatedCount = 0; - while (signalsCreatedCount < tuple.maxSignals) { - try { - let mergedSearchResults = createSearchResultReturnType(); - logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); + if (tuple == null || tuple.to == null || tuple.from == null) { + logger.error(buildRuleMessage(`[-] malformed date tuple`)); + return createSearchAfterReturnType({ + success: false, + errors: ['malformed date tuple'], + }); + } + signalsCreatedCount = 0; + while (signalsCreatedCount < tuple.maxSignals) { + try { + let mergedSearchResults = createSearchResultReturnType(); + logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); - if (hasSortId) { - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ - buildRuleMessage, - searchAfterSortIds: sortIds, - index: inputIndexPattern, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - // @ts-expect-error please, declare a type explicitly instead of unknown - filter, - pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), - timestampOverride: ruleParams.timestampOverride, - trackTotalHits, - sortOrder, - }); - mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnTypeFromResponse({ - searchResult: mergedSearchResults, + if (hasSortId) { + const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + buildRuleMessage, + searchAfterSortIds: sortIds, + index: inputIndexPattern, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + // @ts-expect-error please, declare a type explicitly instead of unknown + filter, + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), timestampOverride: ruleParams.timestampOverride, - }), - createSearchAfterReturnType({ - searchAfterTimes: [searchDuration], - errors: searchErrors, - }), - ]); + trackTotalHits, + sortOrder, + }); + mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); + toReturn = mergeReturns([ + toReturn, + createSearchAfterReturnTypeFromResponse({ + searchResult: mergedSearchResults, + timestampOverride: ruleParams.timestampOverride, + }), + createSearchAfterReturnType({ + searchAfterTimes: [searchDuration], + errors: searchErrors, + }), + ]); - const lastSortIds = getSafeSortIds( - searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort - ); - if (lastSortIds != null && lastSortIds.length !== 0) { - sortIds = lastSortIds; - hasSortId = true; - } else { - hasSortId = false; + const lastSortIds = getSafeSortIds( + searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort + ); + if (lastSortIds != null && lastSortIds.length !== 0) { + sortIds = lastSortIds; + hasSortId = true; + } else { + hasSortId = false; + } } - } - - // determine if there are any candidate signals to be processed - const totalHits = getTotalHitsValue(mergedSearchResults.hits.total); - logger.debug(buildRuleMessage(`totalHits: ${totalHits}`)); - logger.debug( - buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) - ); - if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { + // determine if there are any candidate signals to be processed + const totalHits = getTotalHitsValue(mergedSearchResults.hits.total); + logger.debug(buildRuleMessage(`totalHits: ${totalHits}`)); logger.debug( - buildRuleMessage( - `${ - totalHits === 0 ? 'totalHits' : 'searchResult.hits.hits.length' - } was 0, exiting early` - ) + buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) ); - break; - } - - // filter out the search results that match with the values found in the list. - // the resulting set are signals to be indexed, given they are not duplicates - // of signals already present in the signals index. - const filteredEvents = await filterEventsAgainstList({ - listClient, - exceptionsList, - logger, - eventSearchResult: mergedSearchResults, - buildRuleMessage, - }); - // only bulk create if there are filteredEvents leftover - // if there isn't anything after going through the value list filter - // skip the call to bulk create and proceed to the next search_after, - // if there is a sort id to continue the search_after with. - if (filteredEvents.hits.hits.length !== 0) { - // make sure we are not going to create more signals than maxSignals allows - if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { - filteredEvents.hits.hits = filteredEvents.hits.hits.slice( - 0, - tuple.maxSignals - signalsCreatedCount + if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { + logger.debug( + buildRuleMessage( + `${ + totalHits === 0 ? 'totalHits' : 'searchResult.hits.hits.length' + } was 0, exiting early` + ) ); + break; } - const enrichedEvents = await enrichment(filteredEvents); - const wrappedDocs = wrapHits(enrichedEvents.hits.hits, buildReasonMessage); - const { - bulkCreateDuration: bulkDuration, - createdItemsCount: createdCount, - createdItems, - success: bulkSuccess, - errors: bulkErrors, - } = await bulkCreate(wrappedDocs); + // filter out the search results that match with the values found in the list. + // the resulting set are signals to be indexed, given they are not duplicates + // of signals already present in the signals index. + const filteredEvents = await filterEventsAgainstList({ + listClient, + exceptionsList, + logger, + eventSearchResult: mergedSearchResults, + buildRuleMessage, + }); + + // only bulk create if there are filteredEvents leftover + // if there isn't anything after going through the value list filter + // skip the call to bulk create and proceed to the next search_after, + // if there is a sort id to continue the search_after with. + if (filteredEvents.hits.hits.length !== 0) { + // make sure we are not going to create more signals than maxSignals allows + if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { + filteredEvents.hits.hits = filteredEvents.hits.hits.slice( + 0, + tuple.maxSignals - signalsCreatedCount + ); + } + const enrichedEvents = await enrichment(filteredEvents); + const wrappedDocs = wrapHits(enrichedEvents.hits.hits, buildReasonMessage); - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnType({ + const { + bulkCreateDuration: bulkDuration, + createdItemsCount: createdCount, + createdItems, success: bulkSuccess, - createdSignalsCount: createdCount, - createdSignals: createdItems, - bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, errors: bulkErrors, - }), - ]); - signalsCreatedCount += createdCount; - logger.debug(buildRuleMessage(`created ${createdCount} signals`)); - logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); - logger.debug( - buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) - ); + } = await bulkCreate(wrappedDocs); - sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); - } + toReturn = mergeReturns([ + toReturn, + createSearchAfterReturnType({ + success: bulkSuccess, + createdSignalsCount: createdCount, + createdSignals: createdItems, + bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, + errors: bulkErrors, + }), + ]); + signalsCreatedCount += createdCount; + logger.debug(buildRuleMessage(`created ${createdCount} signals`)); + logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); + logger.debug( + buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) + ); - if (!hasSortId) { - logger.debug(buildRuleMessage('ran out of sort ids to sort on')); - break; + sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); + } + + if (!hasSortId) { + logger.debug(buildRuleMessage('ran out of sort ids to sort on')); + break; + } + } catch (exc: unknown) { + logger.error(buildRuleMessage(`[-] search_after_bulk_create threw an error ${exc}`)); + return mergeReturns([ + toReturn, + createSearchAfterReturnType({ + success: false, + errors: [`${exc}`], + }), + ]); } - } catch (exc: unknown) { - logger.error(buildRuleMessage(`[-] search_after_bulk_create threw an error ${exc}`)); - return mergeReturns([ - toReturn, - createSearchAfterReturnType({ - success: false, - errors: [`${exc}`], - }), - ]); } - } - logger.debug(buildRuleMessage(`[+] completed bulk index of ${toReturn.createdSignalsCount}`)); - return toReturn; + logger.debug(buildRuleMessage(`[+] completed bulk index of ${toReturn.createdSignalsCount}`)); + return toReturn; + }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index f49c6327483e3..4594ce212e0a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,7 +8,6 @@ import { Logger } from 'src/core/server'; import isEmpty from 'lodash/isEmpty'; - import * as t from 'io-ts'; import { validateNonExact, parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; import { SIGNALS_ID } from '@kbn/securitysolution-rules'; @@ -265,6 +264,7 @@ export const signalRulesAlertType = ({ ); logger.warn(gapMessage); hasError = true; + await ruleStatusClient.logStatusChange({ ...basicLogArguments, newStatus: RuleExecutionStatus.failed, @@ -280,6 +280,7 @@ export const signalRulesAlertType = ({ lists, savedObjectClient: services.savedObjectsClient, }); + const exceptionItems = await getExceptions({ client: exceptionsClient, lists: params.exceptionsList ?? [], @@ -402,9 +403,8 @@ export const signalRulesAlertType = ({ wrapSequences, }); } - } else { - throw new Error(`unknown rule type ${type}`); } + if (result.warningMessages.length) { const warningMessage = buildRuleMessage( truncateMessageList(result.warningMessages).join() diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 2596068848ef0..d5a9250621d10 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -17,6 +17,7 @@ import { BuildRuleMessage } from './rule_messages'; import { buildEventsSearchQuery } from './build_events_query'; import { createErrorsFromShard, makeFloatString } from './utils'; import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { withSecuritySpan } from '../../../utils/with_security_span'; interface SingleSearchAfterParams { aggregations?: Record; @@ -54,66 +55,68 @@ export const singleSearchAfter = async ({ searchDuration: string; searchErrors: string[]; }> => { - try { - const searchAfterQuery = buildEventsSearchQuery({ - aggregations, - index, - from, - to, - filter, - size: pageSize, - sortOrder, - searchAfterSortIds, - timestampOverride, - trackTotalHits, - }); + return withSecuritySpan('singleSearchAfter', async () => { + try { + const searchAfterQuery = buildEventsSearchQuery({ + aggregations, + index, + from, + to, + filter, + size: pageSize, + sortOrder, + searchAfterSortIds, + timestampOverride, + trackTotalHits, + }); - const start = performance.now(); - const { body: nextSearchAfterResult } = - await services.scopedClusterClient.asCurrentUser.search( - searchAfterQuery as estypes.SearchRequest - ); - const end = performance.now(); + const start = performance.now(); + const { body: nextSearchAfterResult } = + await services.scopedClusterClient.asCurrentUser.search( + searchAfterQuery as estypes.SearchRequest + ); + const end = performance.now(); - const searchErrors = createErrorsFromShard({ - errors: nextSearchAfterResult._shards.failures ?? [], - }); + const searchErrors = createErrorsFromShard({ + errors: nextSearchAfterResult._shards.failures ?? [], + }); - return { - searchResult: nextSearchAfterResult, - searchDuration: makeFloatString(end - start), - searchErrors, - }; - } catch (exc) { - logger.error(buildRuleMessage(`[-] nextSearchAfter threw an error ${exc}`)); - if ( - exc.message.includes('No mapping found for [@timestamp] in order to sort on') || - exc.message.includes(`No mapping found for [${timestampOverride}] in order to sort on`) - ) { - logger.error(buildRuleMessage(`[-] failure reason: ${exc.message}`)); - - const searchRes: SignalSearchResponse = { - took: 0, - timed_out: false, - _shards: { - total: 1, - successful: 1, - failed: 0, - skipped: 0, - }, - hits: { - total: 0, - max_score: 0, - hits: [], - }, - }; return { - searchResult: searchRes, - searchDuration: '-1.0', - searchErrors: exc.message, + searchResult: nextSearchAfterResult, + searchDuration: makeFloatString(end - start), + searchErrors, }; - } + } catch (exc) { + logger.error(buildRuleMessage(`[-] nextSearchAfter threw an error ${exc}`)); + if ( + exc.message.includes('No mapping found for [@timestamp] in order to sort on') || + exc.message.includes(`No mapping found for [${timestampOverride}] in order to sort on`) + ) { + logger.error(buildRuleMessage(`[-] failure reason: ${exc.message}`)); - throw exc; - } + const searchRes: SignalSearchResponse = { + took: 0, + timed_out: false, + _shards: { + total: 1, + successful: 1, + failed: 0, + skipped: 0, + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + }; + return { + searchResult: searchRes, + searchDuration: '-1.0', + searchErrors: exc.message, + }; + } + + throw exc; + } + }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 8a59d71fe74ec..f814ad9cb14ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -61,6 +61,8 @@ import { import { RACAlert, WrappedRACAlert } from '../rule_types/types'; import { SearchTypes } from '../../../../common/detection_engine/types'; import { IRuleExecutionLogClient } from '../rule_execution_log/types'; +import { withSecuritySpan } from '../../../utils/with_security_span'; + interface SortExceptionsReturn { exceptionsWithValueLists: ExceptionListItemSchema[]; exceptionsWithoutValueLists: ExceptionListItemSchema[]; @@ -217,21 +219,25 @@ export const checkPrivilegesFromEsClient = async ( esClient: ElasticsearchClient, indices: string[] ): Promise => - ( - await esClient.transport.request({ - path: '/_security/user/_has_privileges', - method: 'POST', - body: { - index: [ - { - names: indices ?? [], - allow_restricted_indices: true, - privileges: ['read'], + withSecuritySpan( + 'checkPrivilegesFromEsClient', + async () => + ( + await esClient.transport.request({ + path: '/_security/user/_has_privileges', + method: 'POST', + body: { + index: [ + { + names: indices ?? [], + allow_restricted_indices: true, + privileges: ['read'], + }, + ], }, - ], - }, - }) - ).body as Privilege; + }) + ).body as Privilege + ); export const getNumCatchupIntervals = ({ gap, diff --git a/x-pack/plugins/security_solution/server/utils/with_security_span.ts b/x-pack/plugins/security_solution/server/utils/with_security_span.ts new file mode 100644 index 0000000000000..b0ce360cda68d --- /dev/null +++ b/x-pack/plugins/security_solution/server/utils/with_security_span.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { SpanOptions, withSpan } from '@kbn/apm-utils'; +import agent from 'elastic-apm-node'; +import { APP_ID } from '../../common/constants'; + +type Span = Exclude; + +/** + * This is a thin wrapper around withSpan from @kbn/apm-utils, which sets + * span type to Security APP_ID by default. This span type is used to + * distinguish Security spans from everything else when inspecting traces. + * + * Use this method to capture information about the execution of a specific + * code path and highlight it in APM IU. + * + * @param optionsOrName Span name or span options object + * @param cb Code block you want to measure + * + * @returns Whatever the measured code block returns + */ +export const withSecuritySpan = ( + optionsOrName: SpanOptions | string, + cb: (span?: Span) => Promise +) => + withSpan( + { + type: APP_ID, + ...(typeof optionsOrName === 'string' ? { name: optionsOrName } : optionsOrName), + }, + cb + );