From bec9779523bd8a36aeb7beb9fe74e851ee399f58 Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Mon, 11 Oct 2021 18:41:07 -0600 Subject: [PATCH 1/7] Adds migration for status SO --- .../rule_status_saved_objects_client.ts | 16 +- ...egacy_rule_status_saved_object_mappings.ts | 161 ++++++++++++++++++ .../rules/legacy_rule_status/legacy_utils.ts | 17 ++ .../rule_asset_saved_object_mappings.ts | 32 ++++ .../rules/rule_asset_saved_objects_client.ts | 2 +- .../rules/saved_object_mappings.ts | 79 --------- .../signals/__mocks__/es_results.ts | 3 +- .../security_solution/server/saved_objects.ts | 9 +- .../security_and_spaces/tests/migrations.ts | 27 +++ .../security_solution/migrations/data.json | 36 +++- 10 files changed, 289 insertions(+), 93 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_utils.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts index 720659b72194f..467db1ad8634c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts @@ -12,10 +12,11 @@ import { SavedObjectsUpdateResponse, SavedObjectsFindOptions, SavedObjectsFindResult, + SavedObjectsFindOptionsReference, } from '../../../../../../../../src/core/server'; -import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings'; +// eslint-disable-next-line no-restricted-imports +import { ruleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; import { IRuleStatusSOAttributes } from '../../rules/types'; -import { buildChunkedOrFilter } from '../../signals/utils'; export interface RuleStatusSavedObjectsClient { find: ( @@ -35,7 +36,7 @@ export interface FindBulkResponse { } /** - * @pdeprecated Use RuleExecutionLogClient instead + * @deprecated Use RuleExecutionLogClient instead */ export const ruleStatusSavedObjectsClientFactory = ( savedObjectsClient: SavedObjectsClientContract @@ -51,12 +52,16 @@ export const ruleStatusSavedObjectsClientFactory = ( if (ids.length === 0) { return {}; } - const filter = buildChunkedOrFilter(`${ruleStatusSavedObjectType}.attributes.alertId`, ids); + const filter = `${ruleStatusSavedObjectType}.references.type: "alert"`; + const references = ids.map((alertId) => ({ + id: alertId, + type: 'alert', + })); const order: 'desc' = 'desc'; const aggs = { alertIds: { terms: { - field: `${ruleStatusSavedObjectType}.attributes.alertId`, + field: `${ruleStatusSavedObjectType}.references.id`, size: ids.length, }, aggs: { @@ -77,6 +82,7 @@ export const ruleStatusSavedObjectsClientFactory = ( }; const results = await savedObjectsClient.find({ filter, + hasReference: references, aggs, type: ruleStatusSavedObjectType, perPage: 0, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts new file mode 100644 index 0000000000000..9529e2927db8f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SavedObjectReference, + SavedObjectSanitizedDoc, + SavedObjectsType, + SavedObjectUnsanitizedDoc, +} from 'kibana/server'; +import { isString } from 'lodash/fp'; +import { IRuleSavedAttributesSavedObjectAttributes } from '../types'; +// eslint-disable-next-line no-restricted-imports +import { legacyGetRuleReference } from './legacy_utils'; + +/** + * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and + * additional fields on the Alerting Framework Rule SO. + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + */ +export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status'; + +/** + * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and + * additional fields on the Alerting Framework Rule SO. + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + */ +export const ruleStatusSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + status: { + type: 'keyword', + }, + statusDate: { + type: 'date', + }, + lastFailureAt: { + type: 'date', + }, + lastSuccessAt: { + type: 'date', + }, + lastFailureMessage: { + type: 'text', + }, + lastSuccessMessage: { + type: 'text', + }, + lastLookBackDate: { + type: 'date', + }, + gap: { + type: 'text', + }, + bulkCreateTimeDurations: { + type: 'float', + }, + searchAfterTimeDurations: { + type: 'float', + }, + }, +}; + +/** + * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and + * additional fields on the Alerting Framework Rule SO. + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + */ +export const legacyRuleStatusSavedObjectMigration = { + '7.16.0': ( + doc: SavedObjectUnsanitizedDoc + ): SavedObjectSanitizedDoc => { + return legacyMigrateRuleAlertIdSOReferences(doc); + }, +}; + +/** + * This migrates alertId within legacy `siem-detection-engine-rule-status` to saved object references on an upgrade. + * We only migrate alertId if we find these conditions: + * - alertId is a string and not null, undefined, or malformed data. + * - The existing references do not already have a alertId found within it. + * + * Some of these issues could crop up during either user manual errors of modifying things, earlier migration + * issues, etc... so we are safer to check them as possibilities + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + * @param doc The document having an alertId to migrate into references + * @returns The document migrated with saved object references + */ +export const legacyMigrateRuleAlertIdSOReferences = ( + doc: SavedObjectUnsanitizedDoc +): SavedObjectSanitizedDoc => { + const { references } = doc; + + // Isolate alertId from the doc + const { alertId, ...attributesWithoutAlertId } = doc.attributes; + const existingReferences = references ?? []; + + if (!isString(alertId)) { + // early return if alertId is not a string as expected + return { ...doc, references: existingReferences }; + } else { + const alertReferences = legacyMigrateAlertId({ + alertId, + existingReferences, + }); + + return { + ...doc, + attributes: { + ...attributesWithoutAlertId.attributes, + }, + references: [...existingReferences, ...alertReferences], + }; + } +}; + +/** + * This is a helper to migrate "alertId" + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + * + * @param existingReferences The existing saved object references + * @param alertId The alertId to migrate + * + * @returns The savedObjectReferences migrated + */ +export const legacyMigrateAlertId = ({ + existingReferences, + alertId, +}: { + existingReferences: SavedObjectReference[]; + alertId: string; +}): SavedObjectReference[] => { + const existingReferenceFound = existingReferences.find((reference) => { + return reference.id === alertId && reference.type === 'alert'; + }); + if (existingReferenceFound) { + return []; + } else { + return [legacyGetRuleReference(alertId)]; + } +}; + +/** + * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and + * additional fields on the Alerting Framework Rule SO. + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + */ +export const legacyRuleStatusType: SavedObjectsType = { + name: ruleStatusSavedObjectType, + hidden: false, + namespaceType: 'single', + mappings: ruleStatusSavedObjectMappings, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_utils.ts new file mode 100644 index 0000000000000..62de5ce591230 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_utils.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Given an id this returns a legacy rule reference. + * @param id The id of the alert + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + */ +export const legacyGetRuleReference = (id: string) => ({ + id, + type: 'alert', + name: 'alert_0', +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts new file mode 100644 index 0000000000000..e2941b503664b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsType } from '../../../../../../../../src/core/server'; + +export const ruleAssetSavedObjectType = 'security-rule'; + +export const ruleAssetSavedObjectMappings: SavedObjectsType['mappings'] = { + dynamic: false, + properties: { + name: { + type: 'keyword', + }, + rule_id: { + type: 'keyword', + }, + version: { + type: 'long', + }, + }, +}; + +export const ruleAssetType: SavedObjectsType = { + name: ruleAssetSavedObjectType, + hidden: false, + namespaceType: 'agnostic', + mappings: ruleAssetSavedObjectMappings, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts index ac0969dfc975d..84c2c0403b426 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts @@ -10,7 +10,7 @@ import { SavedObjectsFindOptions, SavedObjectsFindResponse, } from '../../../../../../../src/core/server'; -import { ruleAssetSavedObjectType } from '../rules/saved_object_mappings'; +import { ruleAssetSavedObjectType } from '../rules/rule_asset/rule_asset_saved_object_mappings'; import { IRuleAssetSavedObject } from '../rules/types'; const DEFAULT_PAGE_SIZE = 100; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts deleted file mode 100644 index 813e800f34ce2..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/saved_object_mappings.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SavedObjectsType } from '../../../../../../../src/core/server'; - -export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status'; - -export const ruleStatusSavedObjectMappings: SavedObjectsType['mappings'] = { - properties: { - alertId: { - type: 'keyword', - }, - status: { - type: 'keyword', - }, - statusDate: { - type: 'date', - }, - lastFailureAt: { - type: 'date', - }, - lastSuccessAt: { - type: 'date', - }, - lastFailureMessage: { - type: 'text', - }, - lastSuccessMessage: { - type: 'text', - }, - lastLookBackDate: { - type: 'date', - }, - gap: { - type: 'text', - }, - bulkCreateTimeDurations: { - type: 'float', - }, - searchAfterTimeDurations: { - type: 'float', - }, - }, -}; - -export const type: SavedObjectsType = { - name: ruleStatusSavedObjectType, - hidden: false, - namespaceType: 'single', - mappings: ruleStatusSavedObjectMappings, -}; - -export const ruleAssetSavedObjectType = 'security-rule'; - -export const ruleAssetSavedObjectMappings: SavedObjectsType['mappings'] = { - dynamic: false, - properties: { - name: { - type: 'keyword', - }, - rule_id: { - type: 'keyword', - }, - version: { - type: 'long', - }, - }, -}; - -export const ruleAssetType: SavedObjectsType = { - name: ruleAssetSavedObjectType, - hidden: false, - namespaceType: 'agnostic', - mappings: ruleAssetSavedObjectMappings, -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 207ea497c7e8e..32769b04a866c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -18,7 +18,8 @@ import type { import { SavedObject } from '../../../../../../../../src/core/server'; import { loggingSystemMock } from '../../../../../../../../src/core/server/mocks'; import { IRuleStatusSOAttributes } from '../../rules/types'; -import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings'; +// eslint-disable-next-line no-restricted-imports +import { ruleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { RulesSchema } from '../../../../../common/detection_engine/schemas/response'; import { RuleParams } from '../../schemas/rule_schemas'; diff --git a/x-pack/plugins/security_solution/server/saved_objects.ts b/x-pack/plugins/security_solution/server/saved_objects.ts index 1523b3ddf4cbf..53618d738984b 100644 --- a/x-pack/plugins/security_solution/server/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/saved_objects.ts @@ -8,10 +8,9 @@ import { CoreSetup } from '../../../../src/core/server'; import { noteType, pinnedEventType, timelineType } from './lib/timeline/saved_object_mappings'; -import { - type as ruleStatusType, - ruleAssetType, -} from './lib/detection_engine/rules/saved_object_mappings'; +// eslint-disable-next-line no-restricted-imports +import { legacyRuleStatusType } from './lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; +import { ruleAssetType } from './lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings'; // eslint-disable-next-line no-restricted-imports import { legacyType as legacyRuleActionsType } from './lib/detection_engine/rule_actions/legacy_saved_object_mappings'; import { type as signalsMigrationType } from './lib/detection_engine/migrations/saved_objects'; @@ -24,7 +23,7 @@ const types = [ noteType, pinnedEventType, legacyRuleActionsType, - ruleStatusType, + legacyRuleStatusType, ruleAssetType, timelineType, exceptionsArtifactType, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/migrations.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/migrations.ts index d25fb5bfa5899..7e1687418d0d0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/migrations.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/migrations.ts @@ -65,6 +65,33 @@ export default ({ getService }: FtrProviderContext): void => { undefined ); }); + + it('migrates legacy siem-detection-engine-rule-status to use saved object references', async () => { + const response = await es.get<{ + 'siem-detection-engine-rule-status': { + alertId: string; + }; + references: [{}]; + }>({ + index: '.kibana', + id: 'siem-detection-engine-rule-status:d62d2980-27c4-11ec-92b0-f7b47106bb35', + }); + expect(response.statusCode).to.eql(200); + + // references exist and are expected values + expect(response.body._source?.references).to.eql([ + { + name: 'alert_0', + id: 'fb1046a0-0452-11ec-9b15-d13d79d162f3', + type: 'alert', + }, + ]); + + // alertId no longer exist + expect(response.body._source?.['siem-detection-engine-rule-status'].alertId).to.eql( + undefined + ); + }); }); }); }; diff --git a/x-pack/test/functional/es_archives/security_solution/migrations/data.json b/x-pack/test/functional/es_archives/security_solution/migrations/data.json index 7b8d81135065d..ee857b34555c2 100644 --- a/x-pack/test/functional/es_archives/security_solution/migrations/data.json +++ b/x-pack/test/functional/es_archives/security_solution/migrations/data.json @@ -1,4 +1,5 @@ -{ +[ + { "type": "doc", "value": { "id": "siem-detection-engine-rule-actions:fce024a0-0452-11ec-9b15-d13d79d162f3", @@ -28,4 +29,35 @@ "updated_at": "2021-09-15T22:18:48.369Z" } } -} +}, + { + "type": "doc", + "value": { + "id": "siem-detection-engine-rule-status:d62d2980-27c4-11ec-92b0-f7b47106bb35", + "index": ".kibana_1", + "source": { + "siem-detection-engine-rule-status": { + "alertId": "fb1046a0-0452-11ec-9b15-d13d79d162f3", + "statusDate": "2021-10-11T20:51:26.622Z", + "status": "succeeded", + "lastFailureAt": "2021-10-11T18:10:08.982Z", + "lastSuccessAt": "2021-10-11T20:51:26.622Z", + "lastFailureMessage": "4 days (323690920ms) were not queried between this rule execution and the last execution, so signals may have been missed. Consider increasing your look behind time or adding more Kibana instances. name: \"Threshy\" id: \"d62d2980-27c4-11ec-92b0-f7b47106bb35\" rule id: \"b789c80f-f6d8-41f1-8b4f-b4a23342cde2\" signals index: \".siem-signals-spong-default\"", + "lastSuccessMessage": "succeeded", + "gap": "4 days", + "bulkCreateTimeDurations": [ + "34.49" + ], + "searchAfterTimeDurations": [ + "62.58" + ], + "lastLookBackDate": null + }, + "type": "siem-detection-engine-rule-status", + "references": [], + "coreMigrationVersion": "7.14.0", + "updated_at": "2021-10-11T20:51:26.657Z" + } + } + } + ] From bd30df770be868ef4ecceab746c32d08c501dadf Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Tue, 12 Oct 2021 18:15:33 -0600 Subject: [PATCH 2/7] Updates namings & deprecations, enabled migration, and added support to references for create API --- .../rules/add_prepackaged_rules_route.ts | 2 +- .../get_prepackaged_rules_status_route.ts | 2 +- .../rule_status_saved_objects_client.ts | 29 +++--- .../saved_objects_adapter.ts | 40 +++++--- .../rules/get_prepackaged_rules.ts | 2 +- .../legacy_rule_status/legacy_migrations.ts | 98 +++++++++++++++++++ ...egacy_rule_status_saved_object_mappings.ts | 98 +------------------ .../rule_asset_saved_objects_client.ts | 6 +- .../lib/detection_engine/rules/types.ts | 2 +- .../signals/__mocks__/es_results.ts | 4 +- .../security_solution/migrations/data.json | 2 +- 11 files changed, 155 insertions(+), 130 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_migrations.ts rename x-pack/plugins/security_solution/server/lib/detection_engine/rules/{ => rule_asset}/rule_asset_saved_objects_client.ts (87%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts index 0048c735b0a7c..fed34743e220a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts @@ -31,7 +31,7 @@ import { updatePrepackagedRules } from '../../rules/update_prepacked_rules'; import { getRulesToInstall } from '../../rules/get_rules_to_install'; import { getRulesToUpdate } from '../../rules/get_rules_to_update'; import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules'; -import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client'; +import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset/rule_asset_saved_objects_client'; import { buildSiemResponse } from '../utils'; import { RulesClient } from '../../../../../../alerting/server'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts index 9a06928eee233..a18507eea4977 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts @@ -20,7 +20,7 @@ import { getRulesToUpdate } from '../../rules/get_rules_to_update'; import { findRules } from '../../rules/find_rules'; import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules'; import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules'; -import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client'; +import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset/rule_asset_saved_objects_client'; import { buildFrameworkRequest } from '../../../timeline/utils/common'; import { ConfigType } from '../../../../config'; import { SetupPlugins } from '../../../../plugin'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts index 467db1ad8634c..1f11551f2d91e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts @@ -13,9 +13,10 @@ import { SavedObjectsFindOptions, SavedObjectsFindResult, SavedObjectsFindOptionsReference, -} from '../../../../../../../../src/core/server'; + SavedObjectsCreateOptions, +} from 'kibana/server'; // eslint-disable-next-line no-restricted-imports -import { ruleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; +import { legacyRuleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; import { IRuleStatusSOAttributes } from '../../rules/types'; export interface RuleStatusSavedObjectsClient { @@ -23,7 +24,10 @@ export interface RuleStatusSavedObjectsClient { options?: Omit ) => Promise>>; findBulk: (ids: string[], statusesPerId: number) => Promise; - create: (attributes: IRuleStatusSOAttributes) => Promise>; + create: ( + attributes: IRuleStatusSOAttributes, + options?: SavedObjectsCreateOptions + ) => Promise>; update: ( id: string, attributes: Partial @@ -44,7 +48,7 @@ export const ruleStatusSavedObjectsClientFactory = ( find: async (options) => { const result = await savedObjectsClient.find({ ...options, - type: ruleStatusSavedObjectType, + type: legacyRuleStatusSavedObjectType, }); return result.saved_objects; }, @@ -52,7 +56,6 @@ export const ruleStatusSavedObjectsClientFactory = ( if (ids.length === 0) { return {}; } - const filter = `${ruleStatusSavedObjectType}.references.type: "alert"`; const references = ids.map((alertId) => ({ id: alertId, type: 'alert', @@ -61,7 +64,7 @@ export const ruleStatusSavedObjectsClientFactory = ( const aggs = { alertIds: { terms: { - field: `${ruleStatusSavedObjectType}.references.id`, + field: `${legacyRuleStatusSavedObjectType}.references.id`, size: ids.length, }, aggs: { @@ -69,7 +72,7 @@ export const ruleStatusSavedObjectsClientFactory = ( top_hits: { sort: [ { - [`${ruleStatusSavedObjectType}.statusDate`]: { + [`${legacyRuleStatusSavedObjectType}.statusDate`]: { order, }, }, @@ -81,10 +84,9 @@ export const ruleStatusSavedObjectsClientFactory = ( }, }; const results = await savedObjectsClient.find({ - filter, hasReference: references, aggs, - type: ruleStatusSavedObjectType, + type: legacyRuleStatusSavedObjectType, perPage: 0, }); const buckets = get(results, 'aggregations.alertIds.buckets'); @@ -97,7 +99,10 @@ export const ruleStatusSavedObjectsClientFactory = ( return acc; }, {}); }, - create: (attributes) => savedObjectsClient.create(ruleStatusSavedObjectType, attributes), - update: (id, attributes) => savedObjectsClient.update(ruleStatusSavedObjectType, id, attributes), - delete: (id) => savedObjectsClient.delete(ruleStatusSavedObjectType, id), + create: (attributes, options) => { + return savedObjectsClient.create(legacyRuleStatusSavedObjectType, attributes, options); + }, + update: (id, attributes) => + savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes), + delete: (id) => savedObjectsClient.delete(legacyRuleStatusSavedObjectType, id), }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts index ca806bd58e369..2607a7a3f20e5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts @@ -5,9 +5,12 @@ * 2.0. */ -import { SavedObject } from 'src/core/server'; +import { SavedObject, SavedObjectReference } from 'src/core/server'; import { SavedObjectsClientContract } from '../../../../../../../../src/core/server'; import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; +// eslint-disable-next-line no-restricted-imports +import { legacyGetRuleReference } from '../../rules/legacy_rule_status/legacy_utils'; + import { IRuleStatusSOAttributes } from '../../rules/types'; import { RuleStatusSavedObjectsClient, @@ -79,20 +82,24 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { private createNewRuleStatus = async ( ruleId: string ): Promise> => { + const references: SavedObjectReference[] = [legacyGetRuleReference(ruleId)]; const now = new Date().toISOString(); - return this.ruleStatusClient.create({ - alertId: ruleId, - statusDate: now, - status: RuleExecutionStatus['going to run'], - lastFailureAt: null, - lastSuccessAt: null, - lastFailureMessage: null, - lastSuccessMessage: null, - gap: null, - bulkCreateTimeDurations: [], - searchAfterTimeDurations: [], - lastLookBackDate: null, - }); + return this.ruleStatusClient.create( + { + alertId: ruleId, + statusDate: now, + status: RuleExecutionStatus['going to run'], + lastFailureAt: null, + lastSuccessAt: null, + lastFailureMessage: null, + lastSuccessMessage: null, + gap: null, + bulkCreateTimeDurations: [], + searchAfterTimeDurations: [], + lastLookBackDate: null, + }, + { references } + ); }; private getOrCreateRuleStatuses = async ( @@ -138,7 +145,10 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { // We always update the newest status, so to 'persist' a failure we push a copy to the head of the list await this.ruleStatusClient.update(currentStatus.id, failureAttributes); - const lastStatus = await this.ruleStatusClient.create(failureAttributes); + const references: SavedObjectReference[] = [ + legacyGetRuleReference(failureAttributes.alertId), + ]; + const lastStatus = await this.ruleStatusClient.create(failureAttributes, { references }); // drop oldest failures const oldStatuses = [lastStatus, ...ruleStatuses].slice(MAX_RULE_STATUSES); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts index 6fe326a8d85a3..8116a42f42827 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts @@ -18,7 +18,7 @@ import { // TODO: convert rules files to TS and add explicit type definitions import { rawRules } from './prepackaged_rules'; -import { RuleAssetSavedObjectsClient } from './rule_asset_saved_objects_client'; +import { RuleAssetSavedObjectsClient } from './rule_asset/rule_asset_saved_objects_client'; import { IRuleAssetSOAttributes } from './types'; import { SavedObjectAttributes } from '../../../../../../../src/core/types'; import { ConfigType } from '../../../config'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_migrations.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_migrations.ts new file mode 100644 index 0000000000000..2788e97b7ab00 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_migrations.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SavedObjectReference, + SavedObjectSanitizedDoc, + SavedObjectUnsanitizedDoc, +} from 'kibana/server'; +import { isString } from 'lodash/fp'; +import { IRuleSavedAttributesSavedObjectAttributes } from '../types'; +// eslint-disable-next-line no-restricted-imports +import { legacyGetRuleReference } from './legacy_utils'; + +/** + * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and + * additional fields on the Alerting Framework Rule SO. + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + */ +export const legacyRuleStatusSavedObjectMigration = { + '7.16.0': ( + doc: SavedObjectUnsanitizedDoc + ): SavedObjectSanitizedDoc => { + return legacyMigrateRuleAlertIdSOReferences(doc); + }, +}; + +/** + * This migrates alertId within legacy `siem-detection-engine-rule-status` to saved object references on an upgrade. + * We only migrate alertId if we find these conditions: + * - alertId is a string and not null, undefined, or malformed data. + * - The existing references do not already have a alertId found within it. + * + * Some of these issues could crop up during either user manual errors of modifying things, earlier migration + * issues, etc... so we are safer to check them as possibilities + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + * @param doc The document having an alertId to migrate into references + * @returns The document migrated with saved object references + */ +export const legacyMigrateRuleAlertIdSOReferences = ( + doc: SavedObjectUnsanitizedDoc +): SavedObjectSanitizedDoc => { + const { references } = doc; + + // Isolate alertId from the doc + const { alertId, ...attributesWithoutAlertId } = doc.attributes; + const existingReferences = references ?? []; + + if (!isString(alertId)) { + // early return if alertId is not a string as expected + return { ...doc, references: existingReferences }; + } else { + const alertReferences = legacyMigrateAlertId({ + alertId, + existingReferences, + }); + + return { + ...doc, + attributes: { + ...attributesWithoutAlertId.attributes, + }, + references: [...existingReferences, ...alertReferences], + }; + } +}; + +/** + * This is a helper to migrate "alertId" + * + * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) + * + * @param existingReferences The existing saved object references + * @param alertId The alertId to migrate + * + * @returns The savedObjectReferences migrated + */ +export const legacyMigrateAlertId = ({ + existingReferences, + alertId, +}: { + existingReferences: SavedObjectReference[]; + alertId: string; +}): SavedObjectReference[] => { + const existingReferenceFound = existingReferences.find((reference) => { + return reference.id === alertId && reference.type === 'alert'; + }); + if (existingReferenceFound) { + return []; + } else { + return [legacyGetRuleReference(alertId)]; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts index 9529e2927db8f..3fe3fc06cc7d6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/legacy_rule_status/legacy_rule_status_saved_object_mappings.ts @@ -5,16 +5,9 @@ * 2.0. */ -import { - SavedObjectReference, - SavedObjectSanitizedDoc, - SavedObjectsType, - SavedObjectUnsanitizedDoc, -} from 'kibana/server'; -import { isString } from 'lodash/fp'; -import { IRuleSavedAttributesSavedObjectAttributes } from '../types'; +import { SavedObjectsType } from 'kibana/server'; // eslint-disable-next-line no-restricted-imports -import { legacyGetRuleReference } from './legacy_utils'; +import { legacyRuleStatusSavedObjectMigration } from './legacy_migrations'; /** * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and @@ -22,7 +15,7 @@ import { legacyGetRuleReference } from './legacy_utils'; * * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) */ -export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status'; +export const legacyRuleStatusSavedObjectType = 'siem-detection-engine-rule-status'; /** * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and @@ -65,88 +58,6 @@ export const ruleStatusSavedObjectMappings: SavedObjectsType['mappings'] = { }, }; -/** - * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and - * additional fields on the Alerting Framework Rule SO. - * - * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) - */ -export const legacyRuleStatusSavedObjectMigration = { - '7.16.0': ( - doc: SavedObjectUnsanitizedDoc - ): SavedObjectSanitizedDoc => { - return legacyMigrateRuleAlertIdSOReferences(doc); - }, -}; - -/** - * This migrates alertId within legacy `siem-detection-engine-rule-status` to saved object references on an upgrade. - * We only migrate alertId if we find these conditions: - * - alertId is a string and not null, undefined, or malformed data. - * - The existing references do not already have a alertId found within it. - * - * Some of these issues could crop up during either user manual errors of modifying things, earlier migration - * issues, etc... so we are safer to check them as possibilities - * - * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) - * @param doc The document having an alertId to migrate into references - * @returns The document migrated with saved object references - */ -export const legacyMigrateRuleAlertIdSOReferences = ( - doc: SavedObjectUnsanitizedDoc -): SavedObjectSanitizedDoc => { - const { references } = doc; - - // Isolate alertId from the doc - const { alertId, ...attributesWithoutAlertId } = doc.attributes; - const existingReferences = references ?? []; - - if (!isString(alertId)) { - // early return if alertId is not a string as expected - return { ...doc, references: existingReferences }; - } else { - const alertReferences = legacyMigrateAlertId({ - alertId, - existingReferences, - }); - - return { - ...doc, - attributes: { - ...attributesWithoutAlertId.attributes, - }, - references: [...existingReferences, ...alertReferences], - }; - } -}; - -/** - * This is a helper to migrate "alertId" - * - * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) - * - * @param existingReferences The existing saved object references - * @param alertId The alertId to migrate - * - * @returns The savedObjectReferences migrated - */ -export const legacyMigrateAlertId = ({ - existingReferences, - alertId, -}: { - existingReferences: SavedObjectReference[]; - alertId: string; -}): SavedObjectReference[] => { - const existingReferenceFound = existingReferences.find((reference) => { - return reference.id === alertId && reference.type === 'alert'; - }); - if (existingReferenceFound) { - return []; - } else { - return [legacyGetRuleReference(alertId)]; - } -}; - /** * This side-car rule status SO is deprecated and is to be replaced by the RuleExecutionLog on Event-Log and * additional fields on the Alerting Framework Rule SO. @@ -154,8 +65,9 @@ export const legacyMigrateAlertId = ({ * @deprecated Remove this once we've fully migrated to event-log and no longer require addition status SO (8.x) */ export const legacyRuleStatusType: SavedObjectsType = { - name: ruleStatusSavedObjectType, + name: legacyRuleStatusSavedObjectType, hidden: false, namespaceType: 'single', mappings: ruleStatusSavedObjectMappings, + migrations: legacyRuleStatusSavedObjectMigration, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_objects_client.ts similarity index 87% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_objects_client.ts index 84c2c0403b426..c594385dce22b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_objects_client.ts @@ -9,9 +9,9 @@ import { SavedObjectsClientContract, SavedObjectsFindOptions, SavedObjectsFindResponse, -} from '../../../../../../../src/core/server'; -import { ruleAssetSavedObjectType } from '../rules/rule_asset/rule_asset_saved_object_mappings'; -import { IRuleAssetSavedObject } from '../rules/types'; +} from 'kibana/server'; +import { ruleAssetSavedObjectType } from './rule_asset_saved_object_mappings'; +import { IRuleAssetSavedObject } from '../types'; const DEFAULT_PAGE_SIZE = 100; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 8adf19a53f92b..b851eb33d9d43 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -111,7 +111,7 @@ export type RuleAlertType = SanitizedAlert; // eslint-disable-next-line @typescript-eslint/no-explicit-any export interface IRuleStatusSOAttributes extends Record { - alertId: string; // created alert id. + alertId: string; // created alert id. // TODO: Remove? statusDate: StatusDate; lastFailureAt: LastFailureAt | null | undefined; lastFailureMessage: LastFailureMessage | null | undefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 32769b04a866c..068e5890aad45 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -19,7 +19,7 @@ import { SavedObject } from '../../../../../../../../src/core/server'; import { loggingSystemMock } from '../../../../../../../../src/core/server/mocks'; import { IRuleStatusSOAttributes } from '../../rules/types'; // eslint-disable-next-line no-restricted-imports -import { ruleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; +import { legacyRuleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { RulesSchema } from '../../../../../common/detection_engine/schemas/response'; import { RuleParams } from '../../schemas/rule_schemas'; @@ -726,7 +726,7 @@ export const sampleRuleGuid = '04128c15-0d1b-4716-a4c5-46997ac7f3bd'; export const sampleIdGuid = 'e1e08ddc-5e37-49ff-a258-5393aa44435a'; export const exampleRuleStatus: () => SavedObject = () => ({ - type: ruleStatusSavedObjectType, + type: legacyRuleStatusSavedObjectType, id: '042e6d90-7069-11ea-af8b-0f8ae4fa817e', attributes: { alertId: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', diff --git a/x-pack/test/functional/es_archives/security_solution/migrations/data.json b/x-pack/test/functional/es_archives/security_solution/migrations/data.json index ee857b34555c2..cddc33058339e 100644 --- a/x-pack/test/functional/es_archives/security_solution/migrations/data.json +++ b/x-pack/test/functional/es_archives/security_solution/migrations/data.json @@ -42,7 +42,7 @@ "status": "succeeded", "lastFailureAt": "2021-10-11T18:10:08.982Z", "lastSuccessAt": "2021-10-11T20:51:26.622Z", - "lastFailureMessage": "4 days (323690920ms) were not queried between this rule execution and the last execution, so signals may have been missed. Consider increasing your look behind time or adding more Kibana instances. name: \"Threshy\" id: \"d62d2980-27c4-11ec-92b0-f7b47106bb35\" rule id: \"b789c80f-f6d8-41f1-8b4f-b4a23342cde2\" signals index: \".siem-signals-spong-default\"", + "lastFailureMessage": "4 days (323690920ms) were not queried between this rule execution and the last execution, so signals may have been missed. Consider increasing your look behind time or adding more Kibana instances. name: \"Threshy\" id: \"fb1046a0-0452-11ec-9b15-d13d79d162f3\" rule id: \"b789c80f-f6d8-41f1-8b4f-b4a23342cde2\" signals index: \".siem-signals-spong-default\"", "lastSuccessMessage": "succeeded", "gap": "4 days", "bulkCreateTimeDurations": [ From 122f015a81d3186711f2bc41736c6869fa1885e2 Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Wed, 13 Oct 2021 20:13:09 -0600 Subject: [PATCH 3/7] Updates so client findBulk query and removes alertId --- .../routes/__mocks__/request_responses.ts | 20 +++-- .../rule_status_saved_objects_client.ts | 79 +++++++++++++------ .../saved_objects_adapter.ts | 16 ++-- .../lib/detection_engine/rules/types.ts | 1 - 4 files changed, 77 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 200246ba1a367..9d1cd3cbca3fb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -479,7 +479,6 @@ export const getRuleExecutionStatuses = (): Array< type: 'my-type', id: 'e0b86950-4e9f-11ea-bdbd-07b56aa159b3', attributes: { - alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bc', statusDate: '2020-02-18T15:26:49.783Z', status: RuleExecutionStatus.succeeded, lastFailureAt: undefined, @@ -492,7 +491,13 @@ export const getRuleExecutionStatuses = (): Array< bulkCreateTimeDurations: ['800.43'], }, score: 1, - references: [], + references: [ + { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bc', + type: 'alert', + name: 'alert_0', + }, + ], updated_at: '2020-02-18T15:26:51.333Z', version: 'WzQ2LDFd', }, @@ -500,7 +505,6 @@ export const getRuleExecutionStatuses = (): Array< type: 'my-type', id: '91246bd0-5261-11ea-9650-33b954270f67', attributes: { - alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08', statusDate: '2020-02-18T15:15:58.806Z', status: RuleExecutionStatus.failed, lastFailureAt: '2020-02-18T15:15:58.806Z', @@ -514,7 +518,13 @@ export const getRuleExecutionStatuses = (): Array< bulkCreateTimeDurations: ['800.43'], }, score: 1, - references: [], + references: [ + { + id: '1ea5a820-4da1-4e82-92a1-2b43a7bece08', + type: 'alert', + name: 'alert_0', + }, + ], updated_at: '2020-02-18T15:15:58.860Z', version: 'WzMyLDFd', }, @@ -523,7 +533,6 @@ export const getRuleExecutionStatuses = (): Array< export const getFindBulkResultStatus = (): FindBulkExecutionLogResponse => ({ '04128c15-0d1b-4716-a4c5-46997ac7f3bd': [ { - alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', statusDate: '2020-02-18T15:26:49.783Z', status: RuleExecutionStatus.succeeded, lastFailureAt: undefined, @@ -538,7 +547,6 @@ export const getFindBulkResultStatus = (): FindBulkExecutionLogResponse => ({ ], '1ea5a820-4da1-4e82-92a1-2b43a7bece08': [ { - alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08', statusDate: '2020-02-18T15:15:58.806Z', status: RuleExecutionStatus.failed, lastFailureAt: '2020-02-18T15:15:58.806Z', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts index 1f11551f2d91e..df34bebff1c2c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts @@ -14,6 +14,7 @@ import { SavedObjectsFindResult, SavedObjectsFindOptionsReference, SavedObjectsCreateOptions, + SavedObjectsBulkGetObject, } from 'kibana/server'; // eslint-disable-next-line no-restricted-imports import { legacyRuleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; @@ -30,7 +31,8 @@ export interface RuleStatusSavedObjectsClient { ) => Promise>; update: ( id: string, - attributes: Partial + attributes: Partial, + options?: SavedObjectsCreateOptions ) => Promise>; delete: (id: string) => Promise<{}>; } @@ -56,53 +58,78 @@ export const ruleStatusSavedObjectsClientFactory = ( if (ids.length === 0) { return {}; } + // With migration from `alertId` to `references[].id` it's not possible to fetch + // just the most recent RuleStatusSO's in one query as SO.find() API doesn't support + // `reverse_nested` so you can't include the parent. Broken out into two queries, + // first for fetching most recent RuleStatusSO id's, then the objects themself. + // TODO: Still use one query but return all status SO's and filter server side? Perf test? + + // Query 1: Fetch most recent RuleStatusSO _id's const references = ids.map((alertId) => ({ id: alertId, type: 'alert', })); const order: 'desc' = 'desc'; - const aggs = { - alertIds: { - terms: { - field: `${legacyRuleStatusSavedObjectType}.references.id`, - size: ids.length, + const nestedAggs = { + references: { + nested: { + path: `${legacyRuleStatusSavedObjectType}.references`, }, aggs: { - most_recent_statuses: { - top_hits: { - sort: [ - { - [`${legacyRuleStatusSavedObjectType}.statusDate`]: { - order, - }, + alertIds: { + terms: { + field: `${legacyRuleStatusSavedObjectType}.references.id`, + size: ids.length, + }, + aggs: { + most_recent_statuses: { + top_hits: { + sort: [ + { + [`${legacyRuleStatusSavedObjectType}.statusDate`]: { + order, + }, + }, + ], + size: statusesPerId, }, - ], - size: statusesPerId, + }, }, }, }, }, }; - const results = await savedObjectsClient.find({ + const statusIdResults = await savedObjectsClient.find({ hasReference: references, - aggs, + aggs: nestedAggs, type: legacyRuleStatusSavedObjectType, perPage: 0, }); - const buckets = get(results, 'aggregations.alertIds.buckets'); - return buckets.reduce((acc: Record, bucket: unknown) => { - const key = get(bucket, 'key'); - const hits = get(bucket, 'most_recent_statuses.hits.hits'); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const statuses = hits.map((hit: any) => hit._source['siem-detection-engine-rule-status']); - acc[key] = statuses; + const statusIdResultBuckets = get(statusIdResults, 'aggregations.references.alertIds.buckets'); + const ruleStatusIds: string[] = statusIdResultBuckets.map((b: unknown) => { + const hits = get(b, 'most_recent_statuses.hits.hits'); + return get(hits, `[${hits.length - 1}]._id`); // TODO: top_hits agg doesn't appear to be working above + }); + + // Query 2: Retrieve RuleStatusSO objects via `_id`'s + const objects: SavedObjectsBulkGetObject[] = ruleStatusIds.map((id) => ({ + type: legacyRuleStatusSavedObjectType, + id: id.substring(`${legacyRuleStatusSavedObjectType}:`.length), // TODO: Any gotchas here w/ any potential additional share-capable prefixes + })); + const statusResult = await savedObjectsClient.bulkGet(objects); + + const hits = get(statusResult, 'saved_objects'); + return hits.reduce((acc: Record, hit: unknown) => { + const alertId: string = get(hit, `references[0].id`); + // TODO: Fine to not support multi-status now? + acc[alertId] = [get(hit, 'attributes')]; return acc; }, {}); }, create: (attributes, options) => { return savedObjectsClient.create(legacyRuleStatusSavedObjectType, attributes, options); }, - update: (id, attributes) => - savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes), + update: (id, attributes, options) => + savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes, options), delete: (id) => savedObjectsClient.delete(legacyRuleStatusSavedObjectType, id), }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts index 2607a7a3f20e5..6ef546fb47a5c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts @@ -54,7 +54,7 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { sortField: 'statusDate', sortOrder: 'desc', search: ruleId, - searchFields: ['alertId'], + searchFields: ['references.id'], }); } @@ -71,12 +71,17 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { } public async logExecutionMetrics({ ruleId, metrics }: LogExecutionMetricsArgs) { + const references: SavedObjectReference[] = [legacyGetRuleReference(ruleId)]; const [currentStatus] = await this.getOrCreateRuleStatuses(ruleId); - await this.ruleStatusClient.update(currentStatus.id, { - ...currentStatus.attributes, - ...convertMetricFields(metrics), - }); + await this.ruleStatusClient.update( + currentStatus.id, + { + ...currentStatus.attributes, + ...convertMetricFields(metrics), + }, + { references } + ); } private createNewRuleStatus = async ( @@ -86,7 +91,6 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { const now = new Date().toISOString(); return this.ruleStatusClient.create( { - alertId: ruleId, statusDate: now, status: RuleExecutionStatus['going to run'], lastFailureAt: null, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index b851eb33d9d43..b10dd9550acf0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -111,7 +111,6 @@ export type RuleAlertType = SanitizedAlert; // eslint-disable-next-line @typescript-eslint/no-explicit-any export interface IRuleStatusSOAttributes extends Record { - alertId: string; // created alert id. // TODO: Remove? statusDate: StatusDate; lastFailureAt: LastFailureAt | null | undefined; lastFailureMessage: LastFailureMessage | null | undefined; From 2bf2bd988e8aebd5af3e7e6a4333038f75355ed0 Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Thu, 14 Oct 2021 11:26:24 -0600 Subject: [PATCH 4/7] Fixing migration functional test --- .../security_solution/migrations/data.json | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/x-pack/test/functional/es_archives/security_solution/migrations/data.json b/x-pack/test/functional/es_archives/security_solution/migrations/data.json index cddc33058339e..97a2596f9dba1 100644 --- a/x-pack/test/functional/es_archives/security_solution/migrations/data.json +++ b/x-pack/test/functional/es_archives/security_solution/migrations/data.json @@ -1,4 +1,3 @@ -[ { "type": "doc", "value": { @@ -29,35 +28,36 @@ "updated_at": "2021-09-15T22:18:48.369Z" } } -}, - { - "type": "doc", - "value": { - "id": "siem-detection-engine-rule-status:d62d2980-27c4-11ec-92b0-f7b47106bb35", - "index": ".kibana_1", - "source": { - "siem-detection-engine-rule-status": { - "alertId": "fb1046a0-0452-11ec-9b15-d13d79d162f3", - "statusDate": "2021-10-11T20:51:26.622Z", - "status": "succeeded", - "lastFailureAt": "2021-10-11T18:10:08.982Z", - "lastSuccessAt": "2021-10-11T20:51:26.622Z", - "lastFailureMessage": "4 days (323690920ms) were not queried between this rule execution and the last execution, so signals may have been missed. Consider increasing your look behind time or adding more Kibana instances. name: \"Threshy\" id: \"fb1046a0-0452-11ec-9b15-d13d79d162f3\" rule id: \"b789c80f-f6d8-41f1-8b4f-b4a23342cde2\" signals index: \".siem-signals-spong-default\"", - "lastSuccessMessage": "succeeded", - "gap": "4 days", - "bulkCreateTimeDurations": [ - "34.49" - ], - "searchAfterTimeDurations": [ - "62.58" - ], - "lastLookBackDate": null - }, - "type": "siem-detection-engine-rule-status", - "references": [], - "coreMigrationVersion": "7.14.0", - "updated_at": "2021-10-11T20:51:26.657Z" - } +} + +{ + "type": "doc", + "value": { + "id": "siem-detection-engine-rule-status:d62d2980-27c4-11ec-92b0-f7b47106bb35", + "index": ".kibana_1", + "source": { + "siem-detection-engine-rule-status": { + "alertId": "fb1046a0-0452-11ec-9b15-d13d79d162f3", + "statusDate": "2021-10-11T20:51:26.622Z", + "status": "succeeded", + "lastFailureAt": "2021-10-11T18:10:08.982Z", + "lastSuccessAt": "2021-10-11T20:51:26.622Z", + "lastFailureMessage": "4 days (323690920ms) were not queried between this rule execution and the last execution, so signals may have been missed. Consider increasing your look behind time or adding more Kibana instances. name: \"Threshy\" id: \"fb1046a0-0452-11ec-9b15-d13d79d162f3\" rule id: \"b789c80f-f6d8-41f1-8b4f-b4a23342cde2\" signals index: \".siem-signals-spong-default\"", + "lastSuccessMessage": "succeeded", + "gap": "4 days", + "bulkCreateTimeDurations": [ + "34.49" + ], + "searchAfterTimeDurations": [ + "62.58" + ], + "lastLookBackDate": null + }, + "type": "siem-detection-engine-rule-status", + "references": [], + "coreMigrationVersion": "7.14.0", + "updated_at": "2021-10-11T20:51:26.657Z" } } - ] +} + From 503e55bf18d09451883a4d259103000364f9898b Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Thu, 14 Oct 2021 15:54:08 -0600 Subject: [PATCH 5/7] Adds support for reverse_nested to SO aggs and fixes update references bugs --- .../aggregations/aggs_types/bucket_aggs.ts | 5 +- .../lib/detection_engine/routes/utils.test.ts | 2 +- .../event_log_adapter/event_log_adapter.ts | 4 +- .../rule_status_saved_objects_client.ts | 71 ++++++++----------- .../saved_objects_adapter.ts | 24 ++++--- .../rule_execution_log/types.ts | 1 + .../rules/delete_rules.test.ts | 1 - .../lib/detection_engine/rules/enable_rule.ts | 1 + .../signals/__mocks__/es_results.ts | 9 ++- 9 files changed, 58 insertions(+), 60 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts index cf27505e8f073..f85576aa64451 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts @@ -16,6 +16,7 @@ import { sortOrderSchema } from './common_schemas'; * - filter * - histogram * - nested + * - reverse_nested * - terms * * Not implemented: @@ -37,7 +38,6 @@ import { sortOrderSchema } from './common_schemas'; * - parent * - range * - rare_terms - * - reverse_nested * - sampler * - significant_terms * - significant_text @@ -76,6 +76,9 @@ export const bucketAggsSchemas: Record = { nested: s.object({ path: s.string(), }), + reverse_nested: s.object({ + path: s.maybe(s.string()), + }), terms: s.object({ field: s.maybe(s.string()), collect_mode: s.maybe(s.string()), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts index 10472fe1c0a03..573cd1cdc9df6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts @@ -142,7 +142,7 @@ describe.each([ statusTwo.attributes.status = RuleExecutionStatus.failed; const currentStatus = exampleRuleStatus(); const foundRules = [currentStatus.attributes, statusOne.attributes, statusTwo.attributes]; - const res = mergeStatuses(currentStatus.attributes.alertId, foundRules, { + const res = mergeStatuses(currentStatus.references[0].id, foundRules, { 'myfakealertid-8cfac': { current_status: { alert_id: 'myfakealertid-8cfac', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts index 086cc12788a40..a3fb50f1f6b0b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts @@ -42,7 +42,7 @@ export class EventLogAdapter implements IRuleExecutionLogClient { } public async update(args: UpdateExecutionLogArgs) { - const { attributes, spaceId, ruleName, ruleType } = args; + const { attributes, spaceId, ruleId, ruleName, ruleType } = args; await this.savedObjectsAdapter.update(args); @@ -51,7 +51,7 @@ export class EventLogAdapter implements IRuleExecutionLogClient { this.eventLogClient.logStatusChange({ ruleName, ruleType, - ruleId: attributes.alertId, + ruleId, newStatus: attributes.status, spaceId, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts index df34bebff1c2c..a7427baa879e7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts @@ -5,17 +5,16 @@ * 2.0. */ -import { get } from 'lodash'; import { - SavedObjectsClientContract, SavedObject, - SavedObjectsUpdateResponse, + SavedObjectsClientContract, + SavedObjectsCreateOptions, SavedObjectsFindOptions, - SavedObjectsFindResult, SavedObjectsFindOptionsReference, - SavedObjectsCreateOptions, - SavedObjectsBulkGetObject, + SavedObjectsFindResult, + SavedObjectsUpdateResponse, } from 'kibana/server'; +import { get } from 'lodash'; // eslint-disable-next-line no-restricted-imports import { legacyRuleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings'; import { IRuleStatusSOAttributes } from '../../rules/types'; @@ -58,19 +57,12 @@ export const ruleStatusSavedObjectsClientFactory = ( if (ids.length === 0) { return {}; } - // With migration from `alertId` to `references[].id` it's not possible to fetch - // just the most recent RuleStatusSO's in one query as SO.find() API doesn't support - // `reverse_nested` so you can't include the parent. Broken out into two queries, - // first for fetching most recent RuleStatusSO id's, then the objects themself. - // TODO: Still use one query but return all status SO's and filter server side? Perf test? - - // Query 1: Fetch most recent RuleStatusSO _id's const references = ids.map((alertId) => ({ id: alertId, type: 'alert', })); const order: 'desc' = 'desc'; - const nestedAggs = { + const aggs = { references: { nested: { path: `${legacyRuleStatusSavedObjectType}.references`, @@ -82,16 +74,21 @@ export const ruleStatusSavedObjectsClientFactory = ( size: ids.length, }, aggs: { - most_recent_statuses: { - top_hits: { - sort: [ - { - [`${legacyRuleStatusSavedObjectType}.statusDate`]: { - order, - }, + rule_status: { + reverse_nested: {}, + aggs: { + most_recent_statuses: { + top_hits: { + sort: [ + { + [`${legacyRuleStatusSavedObjectType}.statusDate`]: { + order, + }, + }, + ], + size: statusesPerId, }, - ], - size: statusesPerId, + }, }, }, }, @@ -99,30 +96,18 @@ export const ruleStatusSavedObjectsClientFactory = ( }, }, }; - const statusIdResults = await savedObjectsClient.find({ + const results = await savedObjectsClient.find({ hasReference: references, - aggs: nestedAggs, + aggs, type: legacyRuleStatusSavedObjectType, perPage: 0, }); - const statusIdResultBuckets = get(statusIdResults, 'aggregations.references.alertIds.buckets'); - const ruleStatusIds: string[] = statusIdResultBuckets.map((b: unknown) => { - const hits = get(b, 'most_recent_statuses.hits.hits'); - return get(hits, `[${hits.length - 1}]._id`); // TODO: top_hits agg doesn't appear to be working above - }); - - // Query 2: Retrieve RuleStatusSO objects via `_id`'s - const objects: SavedObjectsBulkGetObject[] = ruleStatusIds.map((id) => ({ - type: legacyRuleStatusSavedObjectType, - id: id.substring(`${legacyRuleStatusSavedObjectType}:`.length), // TODO: Any gotchas here w/ any potential additional share-capable prefixes - })); - const statusResult = await savedObjectsClient.bulkGet(objects); - - const hits = get(statusResult, 'saved_objects'); - return hits.reduce((acc: Record, hit: unknown) => { - const alertId: string = get(hit, `references[0].id`); - // TODO: Fine to not support multi-status now? - acc[alertId] = [get(hit, 'attributes')]; + const buckets = get(results, 'aggregations.references.alertIds.buckets'); + return buckets.reduce((acc: Record, bucket: unknown) => { + const key = get(bucket, 'key'); + const hits = get(bucket, 'rule_status.most_recent_statuses.hits.hits'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + acc[key] = hits.map((hit: any) => hit._source[legacyRuleStatusSavedObjectType]); return acc; }, {}); }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts index 6ef546fb47a5c..9db7afce62ee4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts @@ -62,8 +62,9 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { return this.ruleStatusClient.findBulk(ruleIds, logsCount); } - public async update({ id, attributes }: UpdateExecutionLogArgs) { - await this.ruleStatusClient.update(id, attributes); + public async update({ id, attributes, ruleId }: UpdateExecutionLogArgs) { + const references: SavedObjectReference[] = [legacyGetRuleReference(ruleId)]; + await this.ruleStatusClient.update(id, attributes, { references }); } public async delete(id: string) { @@ -123,6 +124,8 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { }; public async logStatusChange({ newStatus, ruleId, message, metrics }: LogStatusChangeArgs) { + const references: SavedObjectReference[] = [legacyGetRuleReference(ruleId)]; + switch (newStatus) { case RuleExecutionStatus['going to run']: case RuleExecutionStatus.succeeded: @@ -130,10 +133,14 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { case RuleExecutionStatus['partial failure']: { const [currentStatus] = await this.getOrCreateRuleStatuses(ruleId); - await this.ruleStatusClient.update(currentStatus.id, { - ...currentStatus.attributes, - ...buildRuleStatusAttributes(newStatus, message, metrics), - }); + await this.ruleStatusClient.update( + currentStatus.id, + { + ...currentStatus.attributes, + ...buildRuleStatusAttributes(newStatus, message, metrics), + }, + { references } + ); return; } @@ -148,10 +155,7 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { }; // We always update the newest status, so to 'persist' a failure we push a copy to the head of the list - await this.ruleStatusClient.update(currentStatus.id, failureAttributes); - const references: SavedObjectReference[] = [ - legacyGetRuleReference(failureAttributes.alertId), - ]; + await this.ruleStatusClient.update(currentStatus.id, failureAttributes, { references }); const lastStatus = await this.ruleStatusClient.create(failureAttributes, { references }); // drop oldest failures diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts index e38f974ddee2e..564145cfc5d1f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts @@ -53,6 +53,7 @@ export interface LogStatusChangeArgs { export interface UpdateExecutionLogArgs { id: string; attributes: IRuleStatusSOAttributes; + ruleId: string; ruleName: string; ruleType: string; spaceId: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts index f8e1f873377a9..2d82cd7f8732a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts @@ -26,7 +26,6 @@ describe('deleteRules', () => { type: '', references: [], attributes: { - alertId: 'alertId', statusDate: '', lastFailureAt: null, lastFailureMessage: null, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts index 2f3d05e0c9586..b75a1b0d80e9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts @@ -44,6 +44,7 @@ export const enableRule = async ({ const currentStatusToDisable = ruleCurrentStatus[0]; await ruleStatusClient.update({ id: currentStatusToDisable.id, + ruleId: rule.id, ruleName: rule.name, ruleType: rule.alertTypeId, attributes: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 068e5890aad45..078d36a99ad17 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -729,7 +729,6 @@ export const exampleRuleStatus: () => SavedObject = () type: legacyRuleStatusSavedObjectType, id: '042e6d90-7069-11ea-af8b-0f8ae4fa817e', attributes: { - alertId: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', statusDate: '2020-03-27T22:55:59.517Z', status: RuleExecutionStatus.succeeded, lastFailureAt: null, @@ -741,7 +740,13 @@ export const exampleRuleStatus: () => SavedObject = () searchAfterTimeDurations: [], lastLookBackDate: null, }, - references: [], + references: [ + { + id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + type: 'alert', + name: 'alert_0', + }, + ], updated_at: '2020-03-27T22:55:59.577Z', version: 'WzgyMiwxXQ==', }); From 5f4133208137d955196b2e6de06f45475682ff26 Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Thu, 14 Oct 2021 16:20:48 -0600 Subject: [PATCH 6/7] Making create/update options param mandatory to ensure correct document is always updated/created --- .../saved_objects_adapter/rule_status_saved_objects_client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts index a7427baa879e7..66b646e96ea53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts @@ -26,12 +26,12 @@ export interface RuleStatusSavedObjectsClient { findBulk: (ids: string[], statusesPerId: number) => Promise; create: ( attributes: IRuleStatusSOAttributes, - options?: SavedObjectsCreateOptions + options: SavedObjectsCreateOptions ) => Promise>; update: ( id: string, attributes: Partial, - options?: SavedObjectsCreateOptions + options: SavedObjectsCreateOptions ) => Promise>; delete: (id: string) => Promise<{}>; } From ddccdfe524d4d5316560603cba238c9bcc2eec0c Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Thu, 14 Oct 2021 17:10:33 -0600 Subject: [PATCH 7/7] Increases happiness of test input parameters --- .../server/lib/detection_engine/routes/utils.test.ts | 6 +----- .../server/lib/detection_engine/rules/types.ts | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts index 573cd1cdc9df6..6ddeeaa5ea1c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts @@ -136,6 +136,7 @@ describe.each([ describe('mergeStatuses', () => { it('merges statuses and converts from camelCase saved object to snake_case HTTP response', () => { + // const statusOne = exampleRuleStatus(); statusOne.attributes.status = RuleExecutionStatus.failed; const statusTwo = exampleRuleStatus(); @@ -145,7 +146,6 @@ describe.each([ const res = mergeStatuses(currentStatus.references[0].id, foundRules, { 'myfakealertid-8cfac': { current_status: { - alert_id: 'myfakealertid-8cfac', status_date: '2020-03-27T22:55:59.517Z', status: RuleExecutionStatus.succeeded, last_failure_at: null, @@ -163,7 +163,6 @@ describe.each([ expect(res).toEqual({ 'myfakealertid-8cfac': { current_status: { - alert_id: 'myfakealertid-8cfac', status_date: '2020-03-27T22:55:59.517Z', status: 'succeeded', last_failure_at: null, @@ -179,7 +178,6 @@ describe.each([ }, 'f4b8e31d-cf93-4bde-a265-298bde885cd7': { current_status: { - alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', status_date: '2020-03-27T22:55:59.517Z', status: 'succeeded', last_failure_at: null, @@ -193,7 +191,6 @@ describe.each([ }, failures: [ { - alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', status_date: '2020-03-27T22:55:59.517Z', status: 'failed', last_failure_at: null, @@ -206,7 +203,6 @@ describe.each([ last_look_back_date: null, // NOTE: This is no longer used on the UI, but left here in case users are using it within the API }, { - alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', status_date: '2020-03-27T22:55:59.517Z', status: 'failed', last_failure_at: null, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index b10dd9550acf0..53a83d61da78d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -124,7 +124,6 @@ export interface IRuleStatusSOAttributes extends Record { } export interface IRuleStatusResponseAttributes { - alert_id: string; // created alert id. status_date: StatusDate; last_failure_at: LastFailureAt | null | undefined; last_failure_message: LastFailureMessage | null | undefined;