From ebc623a6cd1cdb3a056f5119baba0bcf3ec84314 Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Wed, 8 Mar 2023 13:31:03 +0100 Subject: [PATCH] [Security Solution] Prebuilt rule upgrade and installation endpoints, initial implementation (#148392) **Addresses:** https://github.com/elastic/kibana/issues/148181, https://github.com/elastic/kibana/issues/148182, https://github.com/elastic/kibana/issues/148185 **Partially addresses:** https://github.com/elastic/kibana/issues/148183, https://github.com/elastic/kibana/issues/148189 ## Summary Based on the [POC](https://github.com/elastic/kibana/pull/144060), this PR adds 4 endpoints for the new upgrade and installation workflows for prebuilt rules: - `GET /internal/detection_engine/prebuilt_rules/status` - `POST /internal/detection_engine/prebuilt_rules/upgrade/_review` - `POST /internal/detection_engine/prebuilt_rules/installation/_review` - `POST /internal/detection_engine/prebuilt_rules/_generate_assets` (temporary helper endpoint for development and testing) The new endpoints are hidden behind a feature flag and can be enabled by the following config setting: ```yaml xpack.securitySolution.enableExperimental: ['prebuiltRulesNewUpgradeAndInstallationWorkflowsEnabled'] ``` ## In the next episodes Will be done later in follow-up PRs: - Implementation of some additional response properties for the `upgrade/_review` endpoint: - https://github.com/elastic/kibana/issues/148183 - Making base versions optional for diff calculation (we need to support this in order to be able to still show diffs for rule assets coming from packages without historical versions): - https://github.com/elastic/kibana/issues/148189 - Further development of the diff algorithm: - https://github.com/elastic/kibana/issues/148191 - Test coverage: - https://github.com/elastic/kibana/issues/148192 --- .../response_schema.ts | 33 +++ .../response_schema.ts | 34 +++ .../review_rule_upgrade/response_schema.ts | 46 +++ .../prebuilt_rules/api/urls.ts | 21 +- .../detection_engine/prebuilt_rules/index.ts | 2 - .../model/diff/diffable_rule/build_schema.ts | 23 ++ .../diffable_rule/diffable_field_types.ts | 141 ++++++++++ .../model/diff/diffable_rule/diffable_rule.ts | 241 ++++++++++++++++ .../model/diff/rule_diff/fields_diff.ts | 16 ++ .../model/diff/rule_diff/rule_diff.ts | 70 +++++ .../diff/three_way_diff/three_way_diff.ts | 114 ++++++++ .../three_way_diff/three_way_diff_outcome.ts | 60 ++++ .../three_way_diff/three_way_merge_outcome.ts | 26 ++ .../prebuilt_rules/model/prebuilt_rule.ts | 43 --- .../rule_schema/model/build_rule_schemas.ts | 15 +- .../common_attributes/misc_attributes.ts | 2 +- .../rule_schema/model/rule_schemas.ts | 28 +- .../common/experimental_features.ts | 6 + .../api/generate_assets/route.ts | 119 ++++++++ .../route.test.ts | 38 +-- .../route.ts | 7 +- .../api/get_prebuilt_rules_status/route.ts | 85 ++++++ .../route.test.ts | 65 ++--- .../route.ts | 33 +-- .../prebuilt_rules/api/register_routes.ts | 19 ++ .../api/review_rule_installation/route.ts | 99 +++++++ .../api/review_rule_upgrade/route.ts | 161 +++++++++++ .../detection_engine/prebuilt_rules/index.ts | 4 +- .../logic/diff/calculate_rule_diff.ts | 95 +++++++ .../algorithms/simple_diff_algorithm.ts | 87 ++++++ .../calculation/calculate_rule_fields_diff.ts | 263 ++++++++++++++++++ .../calculation/diff_calculation_helpers.ts | 38 +++ .../normalization/convert_rule_to_diffable.ts | 230 +++++++++++++++ .../extract_building_block_object.ts | 21 ++ .../normalization/extract_rule_data_query.ts | 60 ++++ .../normalization/extract_rule_data_source.ts | 34 +++ .../extract_rule_name_override_object.ts | 21 ++ .../normalization/extract_rule_schedule.ts | 86 ++++++ .../extract_timeline_template_reference.ts | 22 ++ .../extract_timestamp_override_object.ts | 22 ++ .../logic/get_latest_prebuilt_rules.ts | 52 ---- .../logic/get_rules_to_install.test.ts | 57 ++-- .../logic/get_rules_to_install.ts | 8 +- .../logic/get_rules_to_update.test.ts | 259 ++++++++--------- .../logic/get_rules_to_update.ts | 12 +- .../rule_asset_saved_objects_client.ts | 78 ------ .../prebuilt_rule_assets_client.ts | 177 ++++++++++++ .../prebuilt_rule_assets_type.ts} | 10 +- .../prebuilt_rule_assets_validation.ts | 36 +++ .../create_prebuilt_rules.ts | 12 +- .../prebuilt_rule_objects_client.ts | 48 ++++ .../update_prebuilt_rules.test.ts | 41 +-- .../update_prebuilt_rules.ts | 34 +-- .../prebuilt_rules/logic/utils.ts | 10 - .../detection_engine/prebuilt_rules/mocks.ts | 2 +- .../rule_assets/prebuilt_rule_asset.mock.ts} | 8 +- .../rule_assets/prebuilt_rule_asset.test.ts} | 248 ++++++++--------- .../model/rule_assets/prebuilt_rule_asset.ts | 51 ++++ ...le_asset_validate_type_dependents.test.ts} | 20 +- ...lt_rule_asset_validate_type_dependents.ts} | 10 +- .../rule_versions/get_version_buckets.ts | 54 ++++ .../prebuilt_rule_version_info.ts | 16 ++ .../security_solution/server/routes/index.ts | 2 +- .../security_solution/server/saved_objects.ts | 4 +- .../create_prebuilt_rule_saved_objects.ts | 6 +- 65 files changed, 3116 insertions(+), 669 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/response_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_installation/response_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_upgrade/response_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/build_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/fields_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/rule_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff_outcome.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_merge_outcome.ts delete mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/generate_assets/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculate_rule_diff.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/simple_diff_algorithm.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/diff_calculation_helpers.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_building_block_object.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_query.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_source.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_name_override_object.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_schedule.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timeline_template_reference.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timestamp_override_object.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_latest_prebuilt_rules.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_objects_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts rename x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/{rule_asset/rule_asset_saved_object_mappings.ts => rule_assets/prebuilt_rule_assets_type.ts} (70%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_validation.ts rename x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/{ => rule_objects}/create_prebuilt_rules.ts (72%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts rename x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/{ => rule_objects}/update_prebuilt_rules.test.ts (70%) rename x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/{ => rule_objects}/update_prebuilt_rules.ts (73%) rename x-pack/plugins/security_solution/{common => server/lib}/detection_engine/prebuilt_rules/mocks.ts (80%) rename x-pack/plugins/security_solution/{common/detection_engine/prebuilt_rules/model/prebuilt_rule.mock.ts => server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.mock.ts} (88%) rename x-pack/plugins/security_solution/{common/detection_engine/prebuilt_rules/model/prebuilt_rule.test.ts => server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts} (81%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts rename x-pack/plugins/security_solution/{common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.test.ts => server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.test.ts} (87%) rename x-pack/plugins/security_solution/{common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.ts => server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.ts} (85%) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/get_version_buckets.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/prebuilt_rule_version_info.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/response_schema.ts new file mode 100644 index 0000000000000..ea131df38375d --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/response_schema.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface GetPrebuiltRulesStatusResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all prebuilt rules */ + stats: PrebuiltRulesStatusStats; + }; +} + +export interface PrebuiltRulesStatusStats { + /** Total number of existing (known) prebuilt rules */ + num_prebuilt_rules_total: number; + + /** Number of installed prebuilt rules */ + num_prebuilt_rules_installed: number; + + /** Number of prebuilt rules available for installation (not yet installed) */ + num_prebuilt_rules_to_install: number; + + /** Number of installed prebuilt rules available for upgrade (stock + customized) */ + num_prebuilt_rules_to_upgrade: number; + + // In the future we could add more stats such as: + // - number of installed prebuilt rules which were deprecated + // - number of installed prebuilt rules which are not compatible with the current version of Kibana +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_installation/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_installation/response_schema.ts new file mode 100644 index 0000000000000..999e58c524883 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_installation/response_schema.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSignatureId, RuleTagArray, RuleVersion } from '../../../rule_schema'; +import type { DiffableRule } from '../../model/diff/diffable_rule/diffable_rule'; + +export interface ReviewRuleInstallationResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all rules available for installation */ + stats: RuleInstallationStatsForReview; + + /** Info about individual rules: one object per each rule available for installation */ + rules: RuleInstallationInfoForReview[]; + }; +} + +export interface RuleInstallationStatsForReview { + /** Number of prebuilt rules available for installation */ + num_rules_to_install: number; + + /** A union of all tags of all rules available for installation */ + tags: RuleTagArray; +} + +export type RuleInstallationInfoForReview = DiffableRule & { + rule_id: RuleSignatureId; + version: RuleVersion; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_upgrade/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_upgrade/response_schema.ts new file mode 100644 index 0000000000000..97a4cee4c993f --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_upgrade/response_schema.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleObjectId, RuleSignatureId, RuleTagArray } from '../../../rule_schema'; +import type { DiffableRule } from '../../model/diff/diffable_rule/diffable_rule'; +import type { PartialRuleDiff } from '../../model/diff/rule_diff/rule_diff'; + +export interface ReviewRuleUpgradeResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all rules available for upgrade */ + stats: RuleUpgradeStatsForReview; + + /** Info about individual rules: one object per each rule available for upgrade */ + rules: RuleUpgradeInfoForReview[]; + }; +} + +export interface RuleUpgradeStatsForReview { + /** Number of installed prebuilt rules available for upgrade (stock + customized) */ + num_rules_to_upgrade_total: number; + + /** Number of installed prebuilt rules available for upgrade which are stock (non-customized) */ + num_rules_to_upgrade_not_customized: number; + + /** Number of installed prebuilt rules available for upgrade which are customized by the user */ + num_rules_to_upgrade_customized: number; + + /** A union of all tags of all rules available for upgrade */ + tags: RuleTagArray; + + /** A union of all fields "to be upgraded" across all the rules available for upgrade. An array of field names. */ + fields: string[]; +} + +export interface RuleUpgradeInfoForReview { + id: RuleObjectId; + rule_id: RuleSignatureId; + rule: DiffableRule; + diff: PartialRuleDiff; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts index 449960916f239..44727fcf693cf 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts @@ -5,7 +5,22 @@ * 2.0. */ -import { DETECTION_ENGINE_RULES_URL as RULES } from '../../../constants'; +import { + DETECTION_ENGINE_RULES_URL as RULES, + INTERNAL_DETECTION_ENGINE_URL as INTERNAL, +} from '../../../constants'; -export const PREBUILT_RULES_URL = `${RULES}/prepackaged` as const; -export const PREBUILT_RULES_STATUS_URL = `${RULES}/prepackaged/_status` as const; +const OLD_BASE_URL = `${RULES}/prepackaged` as const; +const NEW_BASE_URL = `${INTERNAL}/prebuilt_rules` as const; + +export const PREBUILT_RULES_URL = OLD_BASE_URL; +export const PREBUILT_RULES_STATUS_URL = `${OLD_BASE_URL}/_status` as const; + +export const GET_PREBUILT_RULES_STATUS_URL = `${NEW_BASE_URL}/status` as const; +export const REVIEW_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_review` as const; +export const PERFORM_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_perform` as const; +export const REVIEW_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_review` as const; +export const PERFORM_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_perform` as const; + +// Helper endpoints for development and testing. Should be removed later. +export const GENERATE_ASSETS_URL = `${NEW_BASE_URL}/_generate_assets` as const; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/index.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/index.ts index 5ec8f617f1546..20669682c63f9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/index.ts @@ -8,5 +8,3 @@ export * from './api/get_prebuilt_rules_and_timelines_status/response_schema'; export * from './api/install_prebuilt_rules_and_timelines/response_schema'; export * from './api/urls'; - -export * from './model/prebuilt_rule'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/build_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/build_schema.ts new file mode 100644 index 0000000000000..b0b60f70d1e63 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/build_schema.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { orUndefined } from '../../../../rule_schema/model/build_rule_schemas'; + +interface RuleFields { + required: TRequired; + optional: TOptional; +} + +export const buildSchema = ( + fields: RuleFields +) => { + return t.intersection([ + t.exact(t.type(fields.required)), + t.exact(t.type(orUndefined(fields.optional))), + ]); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types.ts new file mode 100644 index 0000000000000..3e07566afbf96 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { TimeDuration } from '@kbn/securitysolution-io-ts-types'; +import { + BuildingBlockType, + DataViewId, + IndexPatternArray, + KqlQueryLanguage, + RuleFilterArray, + RuleNameOverride as RuleNameOverrideFieldName, + RuleQuery, + TimelineTemplateId, + TimelineTemplateTitle, + TimestampOverride as TimestampOverrideFieldName, + TimestampOverrideFallbackDisabled, +} from '../../../../rule_schema'; +import { saved_id } from '../../../../schemas/common'; + +// ------------------------------------------------------------------------------------------------- +// Rule data source + +export enum DataSourceType { + 'index_patterns' = 'index_patterns', + 'data_view' = 'data_view', +} + +export type DataSourceIndexPatterns = t.TypeOf; +export const DataSourceIndexPatterns = t.exact( + t.type({ + type: t.literal(DataSourceType.index_patterns), + index_patterns: IndexPatternArray, + }) +); + +export type DataSourceDataView = t.TypeOf; +export const DataSourceDataView = t.exact( + t.type({ + type: t.literal(DataSourceType.data_view), + data_view_id: DataViewId, + }) +); + +export type RuleDataSource = t.TypeOf; +export const RuleDataSource = t.union([DataSourceIndexPatterns, DataSourceDataView]); + +// ------------------------------------------------------------------------------------------------- +// Rule data query + +export enum KqlQueryType { + 'inline_query' = 'inline_query', + 'saved_query' = 'saved_query', +} + +export type InlineKqlQuery = t.TypeOf; +export const InlineKqlQuery = t.exact( + t.type({ + type: t.literal(KqlQueryType.inline_query), + query: RuleQuery, + language: KqlQueryLanguage, + filters: RuleFilterArray, + }) +); + +export type SavedKqlQuery = t.TypeOf; +export const SavedKqlQuery = t.exact( + t.type({ + type: t.literal(KqlQueryType.saved_query), + saved_query_id: saved_id, + }) +); + +export type RuleKqlQuery = t.TypeOf; +export const RuleKqlQuery = t.union([InlineKqlQuery, SavedKqlQuery]); + +export type RuleEqlQuery = t.TypeOf; +export const RuleEqlQuery = t.exact( + t.type({ + query: RuleQuery, + language: t.literal('eql'), + filters: RuleFilterArray, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Rule schedule + +export type RuleSchedule = t.TypeOf; +export const RuleSchedule = t.exact( + t.type({ + interval: TimeDuration({ allowedUnits: ['s', 'm', 'h'] }), + lookback: TimeDuration({ allowedUnits: ['s', 'm', 'h'] }), + }) +); + +// ------------------------------------------------------------------------------------------------- +// Rule name override + +export type RuleNameOverrideObject = t.TypeOf; +export const RuleNameOverrideObject = t.exact( + t.type({ + field_name: RuleNameOverrideFieldName, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Timestamp override + +export type TimestampOverrideObject = t.TypeOf; +export const TimestampOverrideObject = t.exact( + t.type({ + field_name: TimestampOverrideFieldName, + fallback_disabled: TimestampOverrideFallbackDisabled, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Reference to a timeline template + +export type TimelineTemplateReference = t.TypeOf; +export const TimelineTemplateReference = t.exact( + t.type({ + timeline_id: TimelineTemplateId, + timeline_title: TimelineTemplateTitle, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Building block + +export type BuildingBlockObject = t.TypeOf; +export const BuildingBlockObject = t.exact( + t.type({ + type: BuildingBlockType, + }) +); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts new file mode 100644 index 0000000000000..81dc7741a2bf4 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +import { + concurrent_searches, + items_per_search, + machine_learning_job_id, + RiskScore, + RiskScoreMapping, + RuleActionArray, + RuleActionThrottle, + Severity, + SeverityMapping, + threat_index, + threat_indicator_path, + threat_mapping, +} from '@kbn/securitysolution-io-ts-alerting-types'; + +import { + AlertSuppression, + EventCategoryOverride, + ExceptionListArray, + HistoryWindowStart, + InvestigationGuide, + MaxSignals, + NewTermsFields, + RelatedIntegrationArray, + RequiredFieldArray, + RuleAuthorArray, + RuleDescription, + RuleFalsePositiveArray, + RuleLicense, + RuleMetadata, + RuleName, + RuleReferenceArray, + RuleSignatureId, + RuleTagArray, + RuleVersion, + SetupGuide, + ThreatArray, + Threshold, + TiebreakerField, + TimestampField, +} from '../../../../rule_schema'; + +import { anomaly_threshold } from '../../../../schemas/common'; + +import { + BuildingBlockObject, + RuleEqlQuery, + InlineKqlQuery, + RuleKqlQuery, + RuleDataSource, + RuleNameOverrideObject, + RuleSchedule, + TimelineTemplateReference, + TimestampOverrideObject, +} from './diffable_field_types'; + +import { buildSchema } from './build_schema'; + +export type DiffableCommonFields = t.TypeOf; +export const DiffableCommonFields = buildSchema({ + required: { + // Technical fields + // NOTE: We might consider removing them from the schema and returning from the API + // not via the fields diff, but via dedicated properties in the response body. + rule_id: RuleSignatureId, + version: RuleVersion, + meta: RuleMetadata, + + // Main domain fields + name: RuleName, + tags: RuleTagArray, + description: RuleDescription, + severity: Severity, + severity_mapping: SeverityMapping, + risk_score: RiskScore, + risk_score_mapping: RiskScoreMapping, + + // About -> Advanced settings + references: RuleReferenceArray, + false_positives: RuleFalsePositiveArray, + threat: ThreatArray, + note: InvestigationGuide, + setup: SetupGuide, + related_integrations: RelatedIntegrationArray, + required_fields: RequiredFieldArray, + author: RuleAuthorArray, + license: RuleLicense, + + // Other domain fields + rule_schedule: RuleSchedule, // NOTE: new field + actions: RuleActionArray, + throttle: RuleActionThrottle, + exceptions_list: ExceptionListArray, + max_signals: MaxSignals, + }, + optional: { + rule_name_override: RuleNameOverrideObject, // NOTE: new field + timestamp_override: TimestampOverrideObject, // NOTE: new field + timeline_template: TimelineTemplateReference, // NOTE: new field + building_block: BuildingBlockObject, // NOTE: new field + }, +}); + +export type DiffableCustomQueryFields = t.TypeOf; +export const DiffableCustomQueryFields = buildSchema({ + required: { + type: t.literal('query'), + data_query: RuleKqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + alert_suppression: AlertSuppression, + }, +}); + +export type DiffableSavedQueryFields = t.TypeOf; +export const DiffableSavedQueryFields = buildSchema({ + required: { + type: t.literal('saved_query'), + data_query: RuleKqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + alert_suppression: AlertSuppression, + }, +}); + +export type DiffableEqlFields = t.TypeOf; +export const DiffableEqlFields = buildSchema({ + required: { + type: t.literal('eql'), + data_query: RuleEqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + event_category_override: EventCategoryOverride, + timestamp_field: TimestampField, + tiebreaker_field: TiebreakerField, + }, +}); + +export type DiffableThreatMatchFields = t.TypeOf; +export const DiffableThreatMatchFields = buildSchema({ + required: { + type: t.literal('threat_match'), + data_query: RuleKqlQuery, // NOTE: new field + threat_query: InlineKqlQuery, // NOTE: new field + threat_index, + threat_mapping, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + threat_indicator_path, + concurrent_searches, // Should combine concurrent_searches and items_per_search? + items_per_search, + }, +}); + +export type DiffableThresholdFields = t.TypeOf; +export const DiffableThresholdFields = buildSchema({ + required: { + type: t.literal('threshold'), + data_query: RuleKqlQuery, // NOTE: new field + threshold: Threshold, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + }, +}); + +export type DiffableMachineLearningFields = t.TypeOf; +export const DiffableMachineLearningFields = buildSchema({ + required: { + type: t.literal('machine_learning'), + machine_learning_job_id, + anomaly_threshold, + }, + optional: {}, +}); + +export type DiffableNewTermsFields = t.TypeOf; +export const DiffableNewTermsFields = buildSchema({ + required: { + type: t.literal('new_terms'), + data_query: InlineKqlQuery, // NOTE: new field + new_terms_fields: NewTermsFields, + history_window_start: HistoryWindowStart, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + }, +}); + +/** + * Represents a normalized rule object that is suitable for passing to the diff algorithm. + * Every top-level field of a diffable rule can be compared separately on its own. + * + * It's important to do such normalization because: + * + * 1. We need to compare installed rules with prebuilt rule content. These objects have similar but not exactly + * the same interfaces. In order to compare them we need to convert them to a common interface. + * + * 2. It only makes sense to compare certain rule fields in combination with other fields. For example, + * we combine `index` and `data_view_id` fields into a `RuleDataSource` object, so that later we could + * calculate a diff for this whole object. If we don't combine them the app would successfully merge the + * following values independently from each other without a conflict: + * + * Base version: index=[logs-*], data_view_id=undefined + * Current version: index=[], data_view_id=some-data-view // user switched to a data view + * Target version: index=[logs-*, filebeat-*], data_view_id=undefined // Elastic added a new index pattern + * Merged version: index=[filebeat-*], data_view_id=some-data-view ??? + * + * Instead, semantically such change represents a conflict because the data source of the rule was changed + * in a potentially incompatible way, and the user might want to review the change and resolve it manually. + * The user must either pick index patterns or a data view, but not both at the same time. + * + * NOTE: Every top-level field in a DiffableRule MUST BE LOGICALLY INDEPENDENT from other + * top-level fields. + */ +export type DiffableRule = t.TypeOf; +export const DiffableRule = t.intersection([ + DiffableCommonFields, + t.union([ + DiffableCustomQueryFields, + DiffableSavedQueryFields, + DiffableEqlFields, + DiffableThreatMatchFields, + DiffableThresholdFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + ]), +]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/fields_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/fields_diff.ts new file mode 100644 index 0000000000000..a1b8a0ae8d104 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/fields_diff.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ThreeWayDiff, ThreeWayDiffAlgorithm } from '../three_way_diff/three_way_diff'; + +export type FieldsDiff = { + [Field in keyof TObject]: ThreeWayDiff; +}; + +export type FieldsDiffAlgorithmsFor = { + [Field in keyof TObject]: ThreeWayDiffAlgorithm; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/rule_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/rule_diff.ts new file mode 100644 index 0000000000000..8a28b358f5303 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/rule_diff/rule_diff.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + DiffableCommonFields, + DiffableCustomQueryFields, + DiffableEqlFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + DiffableSavedQueryFields, + DiffableThreatMatchFields, + DiffableThresholdFields, +} from '../diffable_rule/diffable_rule'; + +import type { FieldsDiff } from './fields_diff'; + +export type CommonFieldsDiff = FieldsDiff; +export type CustomQueryFieldsDiff = FieldsDiff; +export type SavedQueryFieldsDiff = FieldsDiff; +export type EqlFieldsDiff = FieldsDiff; +export type ThreatMatchFieldsDiff = FieldsDiff; +export type ThresholdFieldsDiff = FieldsDiff; +export type MachineLearningFieldsDiff = FieldsDiff; +export type NewTermsFieldsDiff = FieldsDiff; + +/** + * It's an object which keys are the same as keys of DiffableRule, but values are + * three-way diffs calculated for their values. + * + * @example + * { + * name: ThreeWayDiff; + * tags: ThreeWayDiff; + * // etc + * } + */ +export type RuleFieldsDiff = CommonFieldsDiff & + ( + | CustomQueryFieldsDiff + | SavedQueryFieldsDiff + | EqlFieldsDiff + | ThreatMatchFieldsDiff + | ThresholdFieldsDiff + | MachineLearningFieldsDiff + | NewTermsFieldsDiff + ); + +/** + * Full rule diff contains diffs for all the top-level rule fields. + * Even if there's no change at all to a given field, its diff will be included in this object. + * This diff can be useful for internal server-side calculations or debugging. + * Note that this is a pretty large object so returning it from the API might be undesirable. + */ +export interface FullRuleDiff { + fields: RuleFieldsDiff; + has_conflict: boolean; +} + +/** + * Partial rule diff contains diffs only for those rule fields that have some changes to them. + * This diff can be useful for returning info from REST API endpoints because its size is tolerable. + */ +export interface PartialRuleDiff { + fields: Partial; + has_conflict: boolean; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff.ts new file mode 100644 index 0000000000000..521b31dc4ea15 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ThreeWayDiffOutcome } from './three_way_diff_outcome'; +import type { ThreeWayMergeOutcome } from './three_way_merge_outcome'; + +/** + * Three versions of a value to pass to a diff algorithm. + */ +export interface ThreeVersionsOf { + /** + * Corresponds to the stock version of the currently installed prebuilt rule. + */ + base_version: TValue; + + /** + * Corresponds exactly to the currently installed prebuilt rule: + * - to the customized version (if it's customized) + * - to the stock version (if it's not customized) + */ + current_version: TValue; + + /** + * Corresponds to the "new" stock version that the user is trying to upgrade to. + */ + target_version: TValue; +} + +/** + * Represents a result of an abstract three-way diff/merge operation on a value + * (could be a whole rule JSON or a given rule field). + * + * Typical situations: + * + * 1. base=A, current=A, target=A => merged=A, conflict=false + * Stock rule, the value hasn't changed. + * + * 2. base=A, current=A, target=B => merged=B, conflict=false + * Stock rule, the value has changed. + * + * 3. base=A, current=B, target=A => merged=B, conflict=false + * Customized rule, the value hasn't changed. + * + * 4. base=A, current=B, target=B => merged=B, conflict=false + * Customized rule, the value has changed exactly the same way as in the user customization. + * + * 5. base=A, current=B, target=C => merged=D, conflict=false + * Customized rule, the value has changed, conflict between B and C resolved automatically. + * + * 6. base=A, current=B, target=C => merged=C, conflict=true + * Customized rule, the value has changed, conflict between B and C couldn't be resolved automatically. + */ +export interface ThreeWayDiff extends ThreeVersionsOf { + /** + * The result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + * + * Exact merge algorithm depends on the value: + * - one algo could be used for single-line strings and keywords (e.g. rule name) + * - another one could be used for multiline text (e.g. rule description) + * - another one could be used for arrays of keywords (e.g. rule tags) + * - another one could be used for the MITRE ATT&CK data structure + * - etc + * + * Merged version always has a value. We do our best to resolve conflicts automatically. + * If they can't be resolved automatically, merged version is equal to target version. + */ + merged_version: TValue; + + /** + * Tells which combination corresponds to the three input versions of the value for this specific diff. + */ + diff_outcome: ThreeWayDiffOutcome; + + /** + * The type of result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + */ + merge_outcome: ThreeWayMergeOutcome; + + /** + * Tells if the value has changed in the target version and the current version could be updated. + * True if: + * - base=A, current=A, target=B + * - base=A, current=B, target=C + */ + has_update: boolean; + + /** + * True if: + * - current != target and we couldn't automatically resolve the conflict between them + * + * False if: + * - current == target (value won't change) + * - current != target && current == base (stock rule will get a new value) + * - current != target and we automatically resolved the conflict between them + */ + has_conflict: boolean; +} + +/** + * Given the three versions of a value, calculates a three-way diff for it. + */ +export type ThreeWayDiffAlgorithm = ( + versions: ThreeVersionsOf +) => ThreeWayDiff; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff_outcome.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff_outcome.ts new file mode 100644 index 0000000000000..07d3fb526c209 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff_outcome.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEqual } from 'lodash'; + +/** + * Result of comparing three versions of a value against each other. + * Defines 5 typical combinations of 3 versions of a value. + */ +export enum ThreeWayDiffOutcome { + /** Stock rule, the value hasn't changed in the target version. */ + StockValueNoUpdate = 'BASE=A, CURRENT=A, TARGET=A', + + /** Stock rule, the value has changed in the target version. */ + StockValueCanUpdate = 'BASE=A, CURRENT=A, TARGET=B', + + /** Customized rule, the value hasn't changed in the target version comparing to the base one. */ + CustomizedValueNoUpdate = 'BASE=A, CURRENT=B, TARGET=A', + + /** Customized rule, the value has changed in the target version exactly the same way as in the user customization. */ + CustomizedValueSameUpdate = 'BASE=A, CURRENT=B, TARGET=B', + + /** Customized rule, the value has changed in the target version and is not equal to the current version. */ + CustomizedValueCanUpdate = 'BASE=A, CURRENT=B, TARGET=C', +} + +export const determineDiffOutcome = ( + baseVersion: TValue, + currentVersion: TValue, + targetVersion: TValue +): ThreeWayDiffOutcome => { + const baseEqlCurrent = isEqual(baseVersion, currentVersion); + const baseEqlTarget = isEqual(baseVersion, targetVersion); + const currentEqlTarget = isEqual(currentVersion, targetVersion); + + if (baseEqlCurrent) { + return currentEqlTarget + ? ThreeWayDiffOutcome.StockValueNoUpdate + : ThreeWayDiffOutcome.StockValueCanUpdate; + } + + if (baseEqlTarget) { + return ThreeWayDiffOutcome.CustomizedValueNoUpdate; + } + + return currentEqlTarget + ? ThreeWayDiffOutcome.CustomizedValueSameUpdate + : ThreeWayDiffOutcome.CustomizedValueCanUpdate; +}; + +export const determineIfValueCanUpdate = (diffCase: ThreeWayDiffOutcome): boolean => { + return ( + diffCase === ThreeWayDiffOutcome.StockValueCanUpdate || + diffCase === ThreeWayDiffOutcome.CustomizedValueCanUpdate + ); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_merge_outcome.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_merge_outcome.ts new file mode 100644 index 0000000000000..5c2caea0130dd --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_merge_outcome.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Type of result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + */ +export enum ThreeWayMergeOutcome { + /** Took current version and returned as the merged one. */ + Current = 'CURRENT', + + /** Took target version and returned as the merged one. */ + Target = 'TARGET', + + /** Merged three versions successfully into a new one. */ + Merged = 'MERGED', + + /** Couldn't merge three versions because of a conflict. */ + Conflict = 'CONFLICT', +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.ts deleted file mode 100644 index e46615ee992b7..0000000000000 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; -import { - RelatedIntegrationArray, - RequiredFieldArray, - SetupGuide, - RuleSignatureId, - RuleVersion, - BaseCreateProps, - TypeSpecificCreateProps, -} from '../../rule_schema'; - -/** - * Big differences between this schema and the createRulesSchema - * - rule_id is required here - * - version is a required field that must exist - */ -export type PrebuiltRuleToInstall = t.TypeOf; -export const PrebuiltRuleToInstall = t.intersection([ - BaseCreateProps, - TypeSpecificCreateProps, - // version is required in PrebuiltRuleToInstall, so this supercedes the defaultable - // version in baseParams - t.exact( - t.type({ - rule_id: RuleSignatureId, - version: RuleVersion, - }) - ), - t.exact( - t.partial({ - related_integrations: RelatedIntegrationArray, - required_fields: RequiredFieldArray, - setup: SetupGuide, - }) - ), -]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts index f7d52c682d191..c77ba322b1c79 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts @@ -21,6 +21,7 @@ export const buildRuleSchemas = ) => { return { + ...fields, create: buildCreateRuleSchema(fields.required, fields.optional, fields.defaultable), patch: buildPatchRuleSchema(fields.required, fields.optional, fields.defaultable), response: buildResponseRuleSchema(fields.required, fields.optional, fields.defaultable), @@ -59,10 +60,17 @@ const buildPatchRuleSchema = < ]); }; -type OrUndefined

= { +export type OrUndefined

= { [K in keyof P]: P[K] | t.UndefinedC; }; +export const orUndefined =

(props: P): OrUndefined

=> { + return Object.keys(props).reduce((acc, key) => { + acc[key] = t.union([props[key], t.undefined]); + return acc; + }, {}) as OrUndefined

; +}; + export const buildResponseRuleSchema = < Required extends t.Props, Optional extends t.Props, @@ -78,10 +86,7 @@ export const buildResponseRuleSchema = < // the conversion from internal schema to response schema TS will report an error. If we just used t.partial // instead, then optional fields can be accidentally omitted from the conversion - and any actual values // in those fields internally will be stripped in the response. - const optionalWithUndefined = Object.keys(optionalFields).reduce((acc, key) => { - acc[key] = t.union([optionalFields[key], t.undefined]); - return acc; - }, {}) as OrUndefined; + const optionalWithUndefined = orUndefined(optionalFields); return t.intersection([ t.exact(t.type(requiredFields)), t.exact(t.type(optionalWithUndefined)), diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts index 315a15190ec28..c4992da7f2702 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts @@ -47,7 +47,7 @@ export type RuleMetadata = t.TypeOf; export const RuleMetadata = t.object; // should be a more specific type? export type RuleLicense = t.TypeOf; -export const RuleLicense = t.string; // should be non-empty string? +export const RuleLicense = t.string; export type RuleAuthorArray = t.TypeOf; export const RuleAuthorArray = t.array(t.string); // should be non-empty strings? diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts index 5d35811368b39..636c5ee5f6fa8 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts @@ -92,7 +92,7 @@ import { buildRuleSchemas } from './build_rule_schemas'; // ------------------------------------------------------------------------------------------------- // Base schema -const baseSchema = buildRuleSchemas({ +export const baseSchema = buildRuleSchemas({ required: { name: RuleName, description: RuleDescription, @@ -207,10 +207,22 @@ export const SharedResponseProps = t.intersection([ // ------------------------------------------------------------------------------------------------- // EQL rule schema +export enum QueryLanguage { + 'kuery' = 'kuery', + 'lucene' = 'lucene', + 'eql' = 'eql', +} + +export type KqlQueryLanguage = t.TypeOf; +export const KqlQueryLanguage = t.keyof({ kuery: null, lucene: null }); + +export type EqlQueryLanguage = t.TypeOf; +export const EqlQueryLanguage = t.literal('eql'); + const eqlSchema = buildRuleSchemas({ required: { type: t.literal('eql'), - language: t.literal('eql'), + language: EqlQueryLanguage, query: RuleQuery, }, optional: { @@ -257,12 +269,12 @@ const threatMatchSchema = buildRuleSchemas({ saved_id, threat_filters, threat_indicator_path, - threat_language: t.keyof({ kuery: null, lucene: null }), + threat_language: KqlQueryLanguage, concurrent_searches, items_per_search, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -307,7 +319,7 @@ const querySchema = buildRuleSchemas({ }, defaultable: { query: RuleQuery, - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -345,7 +357,7 @@ const savedQuerySchema = buildRuleSchemas({ alert_suppression: AlertSuppression, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -386,7 +398,7 @@ const thresholdSchema = buildRuleSchemas({ saved_id, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -461,7 +473,7 @@ const newTermsSchema = buildRuleSchemas({ filters: RuleFilterArray, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index f1ae3ec9c488d..c90b3220359e3 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -52,6 +52,12 @@ export const allowedExperimentalValues = Object.freeze({ */ extendedRuleExecutionLoggingEnabled: false, + /** + * Enables the new API and UI for https://github.com/elastic/security-team/issues/1974. + * It's a temporary feature flag that will be removed once the feature gets a basic production-ready implementation. + */ + prebuiltRulesNewUpgradeAndInstallationWorkflowsEnabled: false, + /** * Enables the SOC trends timerange and stats on D&R page */ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/generate_assets/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/generate_assets/route.ts new file mode 100644 index 0000000000000..1157fe9d3b9ef --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/generate_assets/route.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import moment from 'moment'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { PositiveIntegerGreaterThanZero } from '@kbn/securitysolution-io-ts-types'; + +import { GENERATE_ASSETS_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../routes/utils'; + +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; +import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; + +type RequestBody = t.TypeOf; +const RequestBody = t.exact( + t.type({ + num_versions_per_rule: PositiveIntegerGreaterThanZero, + }) +); + +/** + * NOTE: This is a helper endpoint for development and testing. It should be removed later. + * This endpoint: + * - reads currently installed latest assets (saved objects of type security-rule) + * - generates more versions of rule assets based on the latest ones (multiple versions per rule) + * - writes the generated saved objects back to the kibana index + */ +export const generateAssetsRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: GENERATE_ASSETS_URL, + validate: { + body: buildRouteValidation(RequestBody), + }, + options: { + tags: ['access:securitySolution'], + timeout: { + // FUNFACT: If we do not add a very long timeout what will happen + // is that Chrome which receive a 408 error and then do a retry. + // This retry can cause lots of connections to happen. Using a very + // long timeout will ensure that Chrome does not do retries and saturate the connections. + idleSocket: moment.duration('1', 'hour').asMilliseconds(), + }, + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core']); + const soClient = ctx.core.savedObjects.client; + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + + const latestRules = await ruleAssetsClient.fetchLatestAssets(); + + const historicalRules = generateHistoricalVersionsForManyRules( + latestRules, + request.body.num_versions_per_rule + ); + + await ruleAssetsClient.bulkCreateAssets(historicalRules); + + return response.ok({ + body: { + num_latest_rules: latestRules.length, + num_installed_versions: historicalRules.length, + }, + }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const generateHistoricalVersionsForManyRules = ( + rules: PrebuiltRuleAsset[], + numberOfVersionsPerRule: number +) => { + const result: PrebuiltRuleAsset[] = []; + + rules.forEach((rule) => { + result.push(...generateHistoricalVersionsForOneRule(rule, numberOfVersionsPerRule)); + }); + + return result; +}; + +const generateHistoricalVersionsForOneRule = ( + rule: PrebuiltRuleAsset, + numberOfVersionsPerRule: number +): PrebuiltRuleAsset[] => { + const { name: ruleName, version: latestVersion, ...restOfRuleAttributes } = rule; + const nextToLatestVersion = latestVersion + 1; + const result: PrebuiltRuleAsset[] = []; + + for (let i = 0; i < numberOfVersionsPerRule; i++) { + const historicalVersion = nextToLatestVersion + i; + result.push({ + name: `${ruleName} v${historicalVersion}`, + version: historicalVersion, + ...restOfRuleAttributes, + }); + } + + return result; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.test.ts index 3d3accea3fa24..64206b257d9f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.test.ts @@ -20,25 +20,29 @@ import { mockCheckTimelinesStatusAfterInstallResult, } from '../../../../timeline/__mocks__/import_timelines'; -jest.mock('../../logic/get_latest_prebuilt_rules', () => { +jest.mock('../../logic/rule_assets/prebuilt_rule_assets_client', () => { return { - getLatestPrebuiltRules: async () => { - return [ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - version: 2, // set one higher than the mocks which is set to 1 to trigger updates + createPrebuiltRuleAssetsClient: () => { + return { + fetchLatestAssets: async () => { + return [ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + version: 2, // set one higher than the mocks which is set to 1 to trigger updates + }, + ]; }, - ]; + }; }, }; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.ts index ced2a0e8ea663..40b3078fe06eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_and_timelines_status/route.ts @@ -18,10 +18,9 @@ import { import { getExistingPrepackagedRules } from '../../../rule_management/logic/search/get_existing_prepackaged_rules'; import { findRules } from '../../../rule_management/logic/search/find_rules'; -import { getLatestPrebuiltRules } from '../../logic/get_latest_prebuilt_rules'; import { getRulesToInstall } from '../../logic/get_rules_to_install'; import { getRulesToUpdate } from '../../logic/get_rules_to_update'; -import { ruleAssetsClientFactory } from '../../logic/rule_asset/rule_asset_saved_objects_client'; +import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; import { rulesToMap } from '../../logic/utils'; import { buildFrameworkRequest } from '../../../../timeline/utils/common'; @@ -47,10 +46,10 @@ export const getPrebuiltRulesAndTimelinesStatusRoute = ( const ctx = await context.resolve(['core', 'alerting']); const savedObjectsClient = ctx.core.savedObjects.client; const rulesClient = ctx.alerting.getRulesClient(); - const ruleAssetsClient = ruleAssetsClientFactory(savedObjectsClient); + const ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); try { - const latestPrebuiltRules = await getLatestPrebuiltRules(ruleAssetsClient); + const latestPrebuiltRules = await ruleAssetsClient.fetchLatestAssets(); const customRules = await findRules({ rulesClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts new file mode 100644 index 0000000000000..1926320114a17 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { GET_PREBUILT_RULES_STATUS_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; +import type { + GetPrebuiltRulesStatusResponseBody, + PrebuiltRulesStatusStats, +} from '../../../../../../common/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/response_schema'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildSiemResponse } from '../../../routes/utils'; + +import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; +import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client'; +import type { VersionBuckets } from '../../model/rule_versions/get_version_buckets'; +import { getVersionBuckets } from '../../model/rule_versions/get_version_buckets'; + +export const getPrebuiltRulesStatusRoute = (router: SecuritySolutionPluginRouter) => { + router.get( + { + path: GET_PREBUILT_RULES_STATUS_URL, + validate: {}, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core', 'alerting']); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); + + const [latestVersions, { installedVersions }] = await Promise.all([ + ruleAssetsClient.fetchLatestVersions(), + ruleObjectsClient.fetchInstalledRules(), + ]); + + const versionBuckets = getVersionBuckets({ + latestVersions, + installedVersions, + }); + + const stats = calculateRuleStats(versionBuckets); + + const body: GetPrebuiltRulesStatusResponseBody = { + status_code: 200, + message: 'OK', + attributes: { + stats, + }, + }; + + return response.ok({ body }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const calculateRuleStats = (buckets: VersionBuckets): PrebuiltRulesStatusStats => { + const { latestVersions, installedVersions, latestVersionsToInstall, installedVersionsToUpgrade } = + buckets; + + return { + num_prebuilt_rules_total: latestVersions.length, + num_prebuilt_rules_installed: installedVersions.length, + num_prebuilt_rules_to_install: latestVersionsToInstall.length, + num_prebuilt_rules_to_upgrade: installedVersionsToUpgrade.length, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.test.ts index f529369438742..af6fca55078fb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.test.ts @@ -13,7 +13,6 @@ import { getBasicEmptySearchResponse, } from '../../../routes/__mocks__/request_responses'; import { requestContextMock, serverMock } from '../../../routes/__mocks__'; -import type { PrebuiltRuleToInstall } from '../../../../../../common/detection_engine/prebuilt_rules'; import { installPrebuiltRulesAndTimelinesRoute, createPrepackagedRules } from './route'; import { listMock } from '@kbn/lists-plugin/server/mocks'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; @@ -32,38 +31,42 @@ jest.mock('../../../rule_management/logic/rule_actions/legacy_action_migration', }; }); -jest.mock('../../logic/get_latest_prebuilt_rules', () => { +jest.mock('../../logic/rule_assets/prebuilt_rule_assets_client', () => { return { - getLatestPrebuiltRules: async (): Promise => { - return [ - { - author: ['Elastic'], - tags: [], - rule_id: 'rule-1', - risk_score: 50, - risk_score_mapping: [], - severity_mapping: [], - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - query: 'user.name: root or user.name: admin', - language: 'kuery', - references: [], - actions: [], - enabled: false, - false_positives: [], - max_signals: 100, - threat: [], - throttle: undefined, - exceptions_list: [], - version: 2, // set one higher than the mocks which is set to 1 to trigger updates + createPrebuiltRuleAssetsClient: () => { + return { + fetchLatestAssets: async () => { + return [ + { + author: ['Elastic'], + tags: [], + rule_id: 'rule-1', + risk_score: 50, + risk_score_mapping: [], + severity_mapping: [], + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + query: 'user.name: root or user.name: admin', + language: 'kuery', + references: [], + actions: [], + enabled: false, + false_positives: [], + max_signals: 100, + threat: [], + throttle: undefined, + exceptions_list: [], + version: 2, // set one higher than the mocks which is set to 1 to trigger updates + }, + ]; }, - ]; + }; }, }; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.ts index e3f8bdff812c7..23209a613f597 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/route.ts @@ -22,12 +22,11 @@ import { import { importTimelineResultSchema } from '../../../../../../common/types/timeline'; import { getExistingPrepackagedRules } from '../../../rule_management/logic/search/get_existing_prepackaged_rules'; -import { getLatestPrebuiltRules } from '../../logic/get_latest_prebuilt_rules'; -import { createPrebuiltRules } from '../../logic/create_prebuilt_rules'; -import { updatePrebuiltRules } from '../../logic/update_prebuilt_rules'; +import { createPrebuiltRules } from '../../logic/rule_objects/create_prebuilt_rules'; +import { updatePrebuiltRules } from '../../logic/rule_objects/update_prebuilt_rules'; import { getRulesToInstall } from '../../logic/get_rules_to_install'; import { getRulesToUpdate } from '../../logic/get_rules_to_update'; -import { ruleAssetsClientFactory } from '../../logic/rule_asset/rule_asset_saved_objects_client'; +import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; import { rulesToMap } from '../../logic/utils'; import { installPrepackagedTimelines } from '../../../../timeline/routes/prepackaged_timelines/install_prepackaged_timelines'; @@ -91,7 +90,7 @@ export const createPrepackagedRules = async ( const savedObjectsClient = context.core.savedObjects.client; const siemClient = context.getAppClient(); const exceptionsListClient = context.getExceptionListClient() ?? exceptionsClient; - const ruleAssetsClient = ruleAssetsClientFactory(savedObjectsClient); + const ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); const { maxTimelineImportExportSize } = config; @@ -104,17 +103,18 @@ export const createPrepackagedRules = async ( await exceptionsListClient.createEndpointList(); } - let latestPrepackagedRulesMap = await getLatestPrebuiltRules(ruleAssetsClient); - if (latestPrepackagedRulesMap.size === 0) { + let latestPrebuiltRules = await ruleAssetsClient.fetchLatestAssets(); + if (latestPrebuiltRules.length === 0) { // Seems no packages with prepackaged rules were installed, try to install the default rules package await installPrebuiltRulesPackage(config, context); // Try to get the prepackaged rules again - latestPrepackagedRulesMap = await getLatestPrebuiltRules(ruleAssetsClient); + latestPrebuiltRules = await ruleAssetsClient.fetchLatestAssets(); } - const installedPrePackagedRules = rulesToMap(await getExistingPrepackagedRules({ rulesClient })); - const rulesToInstall = getRulesToInstall(latestPrepackagedRulesMap, installedPrePackagedRules); - const rulesToUpdate = getRulesToUpdate(latestPrepackagedRulesMap, installedPrePackagedRules); + + const installedPrebuiltRules = rulesToMap(await getExistingPrepackagedRules({ rulesClient })); + const rulesToInstall = getRulesToInstall(latestPrebuiltRules, installedPrebuiltRules); + const rulesToUpdate = getRulesToUpdate(latestPrebuiltRules, installedPrebuiltRules); await createPrebuiltRules(rulesClient, rulesToInstall); @@ -128,14 +128,9 @@ export const createPrepackagedRules = async ( importTimelineResultSchema ); - await updatePrebuiltRules( - rulesClient, - savedObjectsClient, - rulesToUpdate, - context.getRuleExecutionLog() - ); + await updatePrebuiltRules(rulesClient, savedObjectsClient, rulesToUpdate); - const prepackagedRulesOutput: InstallPrebuiltRulesAndTimelinesResponse = { + const prebuiltRulesOutput: InstallPrebuiltRulesAndTimelinesResponse = { rules_installed: rulesToInstall.length, rules_updated: rulesToUpdate.length, timelines_installed: prepackagedTimelinesResult?.timelines_installed ?? 0, @@ -143,7 +138,7 @@ export const createPrepackagedRules = async ( }; const [validated, genericErrors] = validate( - prepackagedRulesOutput, + prebuiltRulesOutput, InstallPrebuiltRulesAndTimelinesResponse ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts index c396eea87da9d..da4bd66748a6b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts @@ -5,16 +5,35 @@ * 2.0. */ +import type { ConfigType } from '../../../../config'; import type { SetupPlugins } from '../../../../plugin_contract'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { getPrebuiltRulesAndTimelinesStatusRoute } from './get_prebuilt_rules_and_timelines_status/route'; +import { getPrebuiltRulesStatusRoute } from './get_prebuilt_rules_status/route'; import { installPrebuiltRulesAndTimelinesRoute } from './install_prebuilt_rules_and_timelines/route'; +import { generateAssetsRoute } from './generate_assets/route'; +import { reviewRuleInstallationRoute } from './review_rule_installation/route'; +import { reviewRuleUpgradeRoute } from './review_rule_upgrade/route'; export const registerPrebuiltRulesRoutes = ( router: SecuritySolutionPluginRouter, + config: ConfigType, security: SetupPlugins['security'] ) => { + const { prebuiltRulesNewUpgradeAndInstallationWorkflowsEnabled } = config.experimentalFeatures; + + // Legacy endpoints that we're going to deprecate getPrebuiltRulesAndTimelinesStatusRoute(router, security); installPrebuiltRulesAndTimelinesRoute(router); + + if (prebuiltRulesNewUpgradeAndInstallationWorkflowsEnabled) { + // New endpoints for the rule upgrade and installation workflows + getPrebuiltRulesStatusRoute(router); + reviewRuleInstallationRoute(router); + reviewRuleUpgradeRoute(router); + + // Helper endpoints for development and testing. Should be removed later. + generateAssetsRoute(router); + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts new file mode 100644 index 0000000000000..74d9c6ee80030 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; + +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; +import { REVIEW_RULE_INSTALLATION_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; +import type { + ReviewRuleInstallationResponseBody, + RuleInstallationInfoForReview, + RuleInstallationStatsForReview, +} from '../../../../../../common/detection_engine/prebuilt_rules/api/review_rule_installation/response_schema'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildSiemResponse } from '../../../routes/utils'; + +import { convertRuleToDiffable } from '../../logic/diff/normalization/convert_rule_to_diffable'; +import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; +import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client'; +import { getVersionBuckets } from '../../model/rule_versions/get_version_buckets'; + +export const reviewRuleInstallationRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: REVIEW_RULE_INSTALLATION_URL, + validate: {}, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core', 'alerting']); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); + + const [latestVersions, { installedVersions }] = await Promise.all([ + ruleAssetsClient.fetchLatestVersions(), + ruleObjectsClient.fetchInstalledRules(), + ]); + + const versionBuckets = getVersionBuckets({ + latestVersions, + installedVersions, + }); + + const rulesToInstall = await ruleAssetsClient.fetchAssetsByVersionInfo( + versionBuckets.latestVersionsToInstall + ); + + const body: ReviewRuleInstallationResponseBody = { + status_code: 200, + message: 'OK', + attributes: { + stats: calculateRuleStats(rulesToInstall), + rules: calculateRuleInfos(rulesToInstall), + }, + }; + + return response.ok({ body }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const getAggregatedTags = (rules: PrebuiltRuleAsset[]): string[] => { + const set = new Set(rules.flatMap((rule) => rule.tags || [])); + return Array.from(set.values()); +}; + +const calculateRuleStats = ( + rulesToInstall: PrebuiltRuleAsset[] +): RuleInstallationStatsForReview => { + const tagsOfRulesToInstall = getAggregatedTags(rulesToInstall); + return { + num_rules_to_install: rulesToInstall.length, + tags: tagsOfRulesToInstall, + }; +}; + +const calculateRuleInfos = ( + rulesToInstall: PrebuiltRuleAsset[] +): RuleInstallationInfoForReview[] => { + return rulesToInstall.map((rule) => convertRuleToDiffable(rule)); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts new file mode 100644 index 0000000000000..3b52cc102110e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pickBy } from 'lodash'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; +import { REVIEW_RULE_UPGRADE_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; +import type { + ReviewRuleUpgradeResponseBody, + RuleUpgradeInfoForReview, + RuleUpgradeStatsForReview, +} from '../../../../../../common/detection_engine/prebuilt_rules/api/review_rule_upgrade/response_schema'; +import type { PrebuiltRuleVersionInfo } from '../../model/rule_versions/prebuilt_rule_version_info'; +import type { + CalculateRuleDiffArgs, + CalculateRuleDiffResult, +} from '../../logic/diff/calculate_rule_diff'; +import { calculateRuleDiff } from '../../logic/diff/calculate_rule_diff'; +import type { ThreeWayDiff } from '../../../../../../common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff'; +import type { RuleResponse } from '../../../../../../common/detection_engine/rule_schema'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildSiemResponse } from '../../../routes/utils'; + +import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; +import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client'; +import { getVersionBuckets } from '../../model/rule_versions/get_version_buckets'; + +export const reviewRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: REVIEW_RULE_UPGRADE_URL, + validate: {}, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core', 'alerting']); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); + + const [latestVersions, { installedVersions, installedRules }] = await Promise.all([ + ruleAssetsClient.fetchLatestVersions(), + ruleObjectsClient.fetchInstalledRules(), + ]); + + const versionBuckets = getVersionBuckets({ + latestVersions, + installedVersions, + }); + + const [baseRules, latestRules] = await Promise.all([ + ruleAssetsClient.fetchAssetsByVersionInfo(versionBuckets.installedVersionsToUpgrade), + ruleAssetsClient.fetchAssetsByVersionInfo(versionBuckets.latestVersionsToUpgrade), + ]); + + const ruleDiffCalculationArgs = getRuleDiffCalculationArgs( + versionBuckets.installedVersionsToUpgrade, + installedRules, + baseRules, + latestRules + ); + const ruleDiffCalculationResults = ruleDiffCalculationArgs.map((args) => { + return calculateRuleDiff(args); + }); + + const body: ReviewRuleUpgradeResponseBody = { + status_code: 200, + message: 'OK', + attributes: { + stats: calculateRuleStats(ruleDiffCalculationResults), + rules: calculateRuleInfos(ruleDiffCalculationResults), + }, + }; + + return response.ok({ body }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const getRuleDiffCalculationArgs = ( + installedVersionsToUpgrade: PrebuiltRuleVersionInfo[], + installedRules: RuleResponse[], + baseRules: PrebuiltRuleAsset[], + latestRules: PrebuiltRuleAsset[] +): CalculateRuleDiffArgs[] => { + const installedRulesMap = new Map(installedRules.map((r) => [r.rule_id, r])); + const baseRulesMap = new Map(baseRules.map((r) => [r.rule_id, r])); + const latestRulesMap = new Map(latestRules.map((r) => [r.rule_id, r])); + + const result: CalculateRuleDiffArgs[] = []; + + installedVersionsToUpgrade.forEach((versionToUpgrade) => { + const ruleId = versionToUpgrade.rule_id; + const installedRule = installedRulesMap.get(ruleId); + const baseRule = baseRulesMap.get(ruleId); + const latestRule = latestRulesMap.get(ruleId); + + // TODO: https://github.com/elastic/kibana/issues/148189 + // Make base versions optional for diff calculation. We need to support this in order to be able + // to still show diffs for rule assets coming from packages without historical versions. + if (installedRule != null && baseRule != null && latestRule != null) { + result.push({ + currentVersion: installedRule, + baseVersion: baseRule, + targetVersion: latestRule, + }); + } + }); + + return result; +}; + +const calculateRuleStats = (results: CalculateRuleDiffResult[]): RuleUpgradeStatsForReview => { + return { + num_rules_to_upgrade_total: results.length, + num_rules_to_upgrade_not_customized: results.length, + num_rules_to_upgrade_customized: 0, + tags: [], + fields: [], + }; +}; + +const calculateRuleInfos = (results: CalculateRuleDiffResult[]): RuleUpgradeInfoForReview[] => { + return results.map((result) => { + const { ruleDiff, ruleVersions } = result; + const installedCurrentVersion = ruleVersions.input.current; + const diffableCurrentVersion = ruleVersions.output.current; + + return { + id: installedCurrentVersion.id, + rule_id: installedCurrentVersion.rule_id, + rule: diffableCurrentVersion, + diff: { + fields: pickBy>( + ruleDiff.fields, + (fieldDiff) => fieldDiff.has_update || fieldDiff.has_conflict + ), + has_conflict: ruleDiff.has_conflict, + }, + }; + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/index.ts index 30a12c10bdb90..1181e5b174480 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/index.ts @@ -6,4 +6,6 @@ */ export { createPrepackagedRules } from './api/install_prebuilt_rules_and_timelines/route'; -export * from './api/register_routes'; +export { registerPrebuiltRulesRoutes } from './api/register_routes'; +export { prebuiltRuleAssetType } from './logic/rule_assets/prebuilt_rule_assets_type'; +export { PrebuiltRuleAsset } from './model/rule_assets/prebuilt_rule_asset'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculate_rule_diff.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculate_rule_diff.ts new file mode 100644 index 0000000000000..026589ad0ca0d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculate_rule_diff.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DiffableRule } from '../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule'; +import type { FullRuleDiff } from '../../../../../../common/detection_engine/prebuilt_rules/model/diff/rule_diff/rule_diff'; +import type { ThreeWayDiff } from '../../../../../../common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff'; +import type { RuleResponse } from '../../../../../../common/detection_engine/rule_schema'; +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; + +import { calculateRuleFieldsDiff } from './calculation/calculate_rule_fields_diff'; +import { convertRuleToDiffable } from './normalization/convert_rule_to_diffable'; + +export interface CalculateRuleDiffArgs { + currentVersion: RuleResponse; + baseVersion: PrebuiltRuleAsset; + targetVersion: PrebuiltRuleAsset; +} + +export interface CalculateRuleDiffResult { + ruleDiff: FullRuleDiff; + ruleVersions: { + input: { + current: RuleResponse; + base: PrebuiltRuleAsset; + target: PrebuiltRuleAsset; + }; + output: { + current: DiffableRule; + base: DiffableRule; + target: DiffableRule; + }; + }; +} + +/** + * Calculates a rule diff for a given set of 3 versions of the rule: + * - currenly installed version + * - base version that is the corresponding stock rule content + * - target version which is the stock rule content the user wants to update the rule to + */ +export const calculateRuleDiff = (args: CalculateRuleDiffArgs): CalculateRuleDiffResult => { + /* + 1. Convert current, base and target versions to `DiffableRule`. + 2. Calculate a `RuleFieldsDiff`. For every top-level field of `DiffableRule`: + 2.1. Pick a code path based on the rule type. + 2.2. Pick a concrete diff algorithm (function) per rule field based on the field name or type. + - one algo for rule name and other simple string fields + - another one for tags and other arrays of keywords + - another one for multiline text fields (investigation guide, setup guide, etc) + - another one for `data_source` + - etc + 2.3. Call the picked diff function to get a `ThreeWayDiff` result + 2.4. Add the result to the `RuleFieldsDiff` object as a key-value pair "fieldName: ThreeWayDiff". + 3. Create and return a result based on the `RuleFieldsDiff`. + */ + + const { baseVersion, currentVersion, targetVersion } = args; + + const diffableBaseVersion = convertRuleToDiffable(baseVersion); + const diffableCurrentVersion = convertRuleToDiffable(currentVersion); + const diffableTargetVersion = convertRuleToDiffable(targetVersion); + + const fieldsDiff = calculateRuleFieldsDiff({ + base_version: diffableBaseVersion, + current_version: diffableCurrentVersion, + target_version: diffableTargetVersion, + }); + + const hasAnyFieldConflict = Object.values>(fieldsDiff).some( + (fieldDiff) => fieldDiff.has_conflict + ); + + return { + ruleDiff: { + fields: fieldsDiff, + has_conflict: hasAnyFieldConflict, + }, + ruleVersions: { + input: { + current: currentVersion, + base: baseVersion, + target: targetVersion, + }, + output: { + current: diffableCurrentVersion, + base: diffableBaseVersion, + target: diffableTargetVersion, + }, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/simple_diff_algorithm.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/simple_diff_algorithm.ts new file mode 100644 index 0000000000000..699f8f20bcf1f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/simple_diff_algorithm.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { assertUnreachable } from '../../../../../../../../common/utility_types'; +import type { + ThreeVersionsOf, + ThreeWayDiff, +} from '../../../../../../../../common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff'; +import { + determineDiffOutcome, + determineIfValueCanUpdate, + ThreeWayDiffOutcome, +} from '../../../../../../../../common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff_outcome'; +import { ThreeWayMergeOutcome } from '../../../../../../../../common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_merge_outcome'; + +export const simpleDiffAlgorithm = ( + versions: ThreeVersionsOf +): ThreeWayDiff => { + const { + base_version: baseVersion, + current_version: currentVersion, + target_version: targetVersion, + } = versions; + + const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); + const valueCanUpdate = determineIfValueCanUpdate(diffOutcome); + + const { mergeOutcome, mergedVersion } = mergeVersions( + baseVersion, + currentVersion, + targetVersion, + diffOutcome + ); + + return { + base_version: baseVersion, + current_version: currentVersion, + target_version: targetVersion, + merged_version: mergedVersion, + + diff_outcome: diffOutcome, + merge_outcome: mergeOutcome, + has_update: valueCanUpdate, + has_conflict: mergeOutcome === ThreeWayMergeOutcome.Conflict, + }; +}; + +interface MergeResult { + mergeOutcome: ThreeWayMergeOutcome; + mergedVersion: TValue; +} + +const mergeVersions = ( + baseVersion: TValue, + currentVersion: TValue, + targetVersion: TValue, + diffOutcome: ThreeWayDiffOutcome +): MergeResult => { + switch (diffOutcome) { + case ThreeWayDiffOutcome.StockValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueSameUpdate: { + return { + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + } + case ThreeWayDiffOutcome.StockValueCanUpdate: { + return { + mergeOutcome: ThreeWayMergeOutcome.Target, + mergedVersion: targetVersion, + }; + } + case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { + return { + mergeOutcome: ThreeWayMergeOutcome.Conflict, + mergedVersion: targetVersion, + }; + } + default: + return assertUnreachable(diffOutcome); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts new file mode 100644 index 0000000000000..bb7a83ff20b23 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts @@ -0,0 +1,263 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { assertUnreachable } from '../../../../../../../common/utility_types'; +import { invariant } from '../../../../../../../common/utils/invariant'; + +import type { + DiffableCommonFields, + DiffableCustomQueryFields, + DiffableEqlFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + DiffableRule, + DiffableSavedQueryFields, + DiffableThreatMatchFields, + DiffableThresholdFields, +} from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule'; +import type { + CommonFieldsDiff, + CustomQueryFieldsDiff, + EqlFieldsDiff, + MachineLearningFieldsDiff, + NewTermsFieldsDiff, + RuleFieldsDiff, + SavedQueryFieldsDiff, + ThreatMatchFieldsDiff, + ThresholdFieldsDiff, +} from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/rule_diff/rule_diff'; + +import type { FieldsDiffAlgorithmsFor } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/rule_diff/fields_diff'; +import type { ThreeVersionsOf } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff'; +import { calculateFieldsDiffFor } from './diff_calculation_helpers'; +import { simpleDiffAlgorithm } from './algorithms/simple_diff_algorithm'; + +const BASE_TYPE_ERROR = `Base version can't be of different rule type`; +const TARGET_TYPE_ERROR = `Target version can't be of different rule type`; + +/** + * Calculates a three-way diff per each top-level rule field. + * Returns an object which keys are equal to rule's field names and values are + * three-way diffs calculated for those fields. + */ +export const calculateRuleFieldsDiff = ( + ruleVersions: ThreeVersionsOf +): RuleFieldsDiff => { + validateRuleVersions(ruleVersions); + + const commonFieldsDiff = calculateCommonFieldsDiff(ruleVersions); + // eslint-disable-next-line @typescript-eslint/naming-convention + const { base_version, current_version, target_version } = ruleVersions; + + switch (current_version.type) { + case 'query': { + invariant(base_version.type === 'query', BASE_TYPE_ERROR); + invariant(target_version.type === 'query', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateCustomQueryFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'saved_query': { + invariant(base_version.type === 'saved_query', BASE_TYPE_ERROR); + invariant(target_version.type === 'saved_query', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateSavedQueryFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'eql': { + invariant(base_version.type === 'eql', BASE_TYPE_ERROR); + invariant(target_version.type === 'eql', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateEqlFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'threat_match': { + invariant(base_version.type === 'threat_match', BASE_TYPE_ERROR); + invariant(target_version.type === 'threat_match', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateThreatMatchFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'threshold': { + invariant(base_version.type === 'threshold', BASE_TYPE_ERROR); + invariant(target_version.type === 'threshold', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateThresholdFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'machine_learning': { + invariant(base_version.type === 'machine_learning', BASE_TYPE_ERROR); + invariant(target_version.type === 'machine_learning', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateMachineLearningFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'new_terms': { + invariant(base_version.type === 'new_terms', BASE_TYPE_ERROR); + invariant(target_version.type === 'new_terms', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateNewTermsFieldsDiff({ base_version, current_version, target_version }), + }; + } + default: { + return assertUnreachable(current_version, 'Unhandled rule type'); + } + } +}; + +const validateRuleVersions = (ruleVersions: ThreeVersionsOf): void => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { base_version, current_version, target_version } = ruleVersions; + const types = new Set([base_version.type, current_version.type, target_version.type]); + + if (types.size > 1) { + throw new Error('Cannot change rule type during rule upgrade'); + } +}; + +const calculateCommonFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): CommonFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, commonFieldsDiffAlgorithms); +}; + +const commonFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + rule_id: simpleDiffAlgorithm, + version: simpleDiffAlgorithm, + meta: simpleDiffAlgorithm, + name: simpleDiffAlgorithm, + tags: simpleDiffAlgorithm, + description: simpleDiffAlgorithm, + severity: simpleDiffAlgorithm, + severity_mapping: simpleDiffAlgorithm, + risk_score: simpleDiffAlgorithm, + risk_score_mapping: simpleDiffAlgorithm, + references: simpleDiffAlgorithm, + false_positives: simpleDiffAlgorithm, + threat: simpleDiffAlgorithm, + note: simpleDiffAlgorithm, + setup: simpleDiffAlgorithm, + related_integrations: simpleDiffAlgorithm, + required_fields: simpleDiffAlgorithm, + author: simpleDiffAlgorithm, + license: simpleDiffAlgorithm, + rule_schedule: simpleDiffAlgorithm, + actions: simpleDiffAlgorithm, + throttle: simpleDiffAlgorithm, + exceptions_list: simpleDiffAlgorithm, + max_signals: simpleDiffAlgorithm, + rule_name_override: simpleDiffAlgorithm, + timestamp_override: simpleDiffAlgorithm, + timeline_template: simpleDiffAlgorithm, + building_block: simpleDiffAlgorithm, +}; + +const calculateCustomQueryFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): CustomQueryFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, customQueryFieldsDiffAlgorithms); +}; + +const customQueryFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + alert_suppression: simpleDiffAlgorithm, +}; + +const calculateSavedQueryFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): SavedQueryFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, savedQueryFieldsDiffAlgorithms); +}; + +const savedQueryFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + alert_suppression: simpleDiffAlgorithm, +}; + +const calculateEqlFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): EqlFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, eqlFieldsDiffAlgorithms); +}; + +const eqlFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + event_category_override: simpleDiffAlgorithm, + timestamp_field: simpleDiffAlgorithm, + tiebreaker_field: simpleDiffAlgorithm, +}; + +const calculateThreatMatchFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): ThreatMatchFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, threatMatchFieldsDiffAlgorithms); +}; + +const threatMatchFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + threat_query: simpleDiffAlgorithm, + threat_index: simpleDiffAlgorithm, + threat_mapping: simpleDiffAlgorithm, + threat_indicator_path: simpleDiffAlgorithm, + concurrent_searches: simpleDiffAlgorithm, + items_per_search: simpleDiffAlgorithm, +}; + +const calculateThresholdFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): ThresholdFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, thresholdFieldsDiffAlgorithms); +}; + +const thresholdFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + threshold: simpleDiffAlgorithm, +}; + +const calculateMachineLearningFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): MachineLearningFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, machineLearningFieldsDiffAlgorithms); +}; + +const machineLearningFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = + { + type: simpleDiffAlgorithm, + machine_learning_job_id: simpleDiffAlgorithm, + anomaly_threshold: simpleDiffAlgorithm, + }; + +const calculateNewTermsFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): NewTermsFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, newTermsFieldsDiffAlgorithms); +}; + +const newTermsFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + new_terms_fields: simpleDiffAlgorithm, + history_window_start: simpleDiffAlgorithm, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/diff_calculation_helpers.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/diff_calculation_helpers.ts new file mode 100644 index 0000000000000..eb1b6e2d38aca --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/diff_calculation_helpers.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues } from 'lodash'; +import type { + FieldsDiff, + FieldsDiffAlgorithmsFor, +} from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/rule_diff/fields_diff'; +import type { ThreeVersionsOf } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff'; + +export const calculateFieldsDiffFor = ( + objectVersions: ThreeVersionsOf, + fieldsDiffAlgorithms: FieldsDiffAlgorithmsFor +): FieldsDiff => { + const result = mapValues(fieldsDiffAlgorithms, (calculateFieldDiff, fieldName) => { + const fieldVersions = pickField(fieldName as keyof TObject, objectVersions); + const fieldDiff = calculateFieldDiff(fieldVersions); + return fieldDiff; + }); + + // TODO: try to improve strict typing and get rid of this "as" operator. + return result as FieldsDiff; +}; + +const pickField = ( + fieldName: keyof TObject, + versions: ThreeVersionsOf +): ThreeVersionsOf => { + return { + base_version: versions.base_version[fieldName], + current_version: versions.current_version[fieldName], + target_version: versions.target_version[fieldName], + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts new file mode 100644 index 0000000000000..f917b7154a163 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DEFAULT_MAX_SIGNALS } from '../../../../../../../common/constants'; +import { assertUnreachable } from '../../../../../../../common/utility_types'; +import type { + EqlRule, + EqlRuleCreateProps, + MachineLearningRule, + MachineLearningRuleCreateProps, + NewTermsRule, + NewTermsRuleCreateProps, + QueryRule, + QueryRuleCreateProps, + RuleResponse, + SavedQueryRule, + SavedQueryRuleCreateProps, + ThreatMatchRule, + ThreatMatchRuleCreateProps, + ThresholdRule, + ThresholdRuleCreateProps, +} from '../../../../../../../common/detection_engine/rule_schema'; +import type { PrebuiltRuleAsset } from '../../../model/rule_assets/prebuilt_rule_asset'; +import type { + DiffableCommonFields, + DiffableCustomQueryFields, + DiffableEqlFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + DiffableRule, + DiffableSavedQueryFields, + DiffableThreatMatchFields, + DiffableThresholdFields, +} from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule'; +import { extractBuildingBlockObject } from './extract_building_block_object'; +import { + extractInlineKqlQuery, + extractRuleEqlQuery, + extractRuleKqlQuery, +} from './extract_rule_data_query'; +import { extractRuleDataSource } from './extract_rule_data_source'; +import { extractRuleNameOverrideObject } from './extract_rule_name_override_object'; +import { extractRuleSchedule } from './extract_rule_schedule'; +import { extractTimelineTemplateReference } from './extract_timeline_template_reference'; +import { extractTimestampOverrideObject } from './extract_timestamp_override_object'; + +/** + * Normalizes a given rule to the form which is suitable for passing to the diff algorithm. + * Read more in the JSDoc description of DiffableRule. + */ +export const convertRuleToDiffable = (rule: RuleResponse | PrebuiltRuleAsset): DiffableRule => { + const commonFields = extractDiffableCommonFields(rule); + + switch (rule.type) { + case 'query': + return { + ...commonFields, + ...extractDiffableCustomQueryFields(rule), + }; + case 'saved_query': + return { + ...commonFields, + ...extractDiffableSavedQueryFieldsFromRuleObject(rule), + }; + case 'eql': + return { + ...commonFields, + ...extractDiffableEqlFieldsFromRuleObject(rule), + }; + case 'threat_match': + return { + ...commonFields, + ...extractDiffableThreatMatchFieldsFromRuleObject(rule), + }; + case 'threshold': + return { + ...commonFields, + ...extractDiffableThresholdFieldsFromRuleObject(rule), + }; + case 'machine_learning': + return { + ...commonFields, + ...extractDiffableMachineLearningFieldsFromRuleObject(rule), + }; + case 'new_terms': + return { + ...commonFields, + ...extractDiffableNewTermsFieldsFromRuleObject(rule), + }; + default: + return assertUnreachable(rule, 'Unhandled rule type'); + } +}; + +const extractDiffableCommonFields = ( + rule: RuleResponse | PrebuiltRuleAsset +): DiffableCommonFields => { + return { + // --------------------- REQUIRED FIELDS + // Technical fields + rule_id: rule.rule_id, + version: rule.version, + meta: rule.meta ?? {}, + + // Main domain fields + name: rule.name, + tags: rule.tags ?? [], + description: rule.description, + severity: rule.severity, + severity_mapping: rule.severity_mapping ?? [], + risk_score: rule.risk_score, + risk_score_mapping: rule.risk_score_mapping ?? [], + + // About -> Advanced settings + references: rule.references ?? [], + false_positives: rule.false_positives ?? [], + threat: rule.threat ?? [], + note: rule.note ?? '', + setup: rule.setup ?? '', + related_integrations: rule.related_integrations ?? [], + required_fields: rule.required_fields ?? [], + author: rule.author ?? [], + license: rule.license ?? '', + + // Other domain fields + rule_schedule: extractRuleSchedule(rule), + actions: rule.actions ?? [], + throttle: rule.throttle ?? 'no_actions', + exceptions_list: rule.exceptions_list ?? [], + max_signals: rule.max_signals ?? DEFAULT_MAX_SIGNALS, + + // --------------------- OPTIONAL FIELDS + rule_name_override: extractRuleNameOverrideObject(rule), + timestamp_override: extractTimestampOverrideObject(rule), + timeline_template: extractTimelineTemplateReference(rule), + building_block: extractBuildingBlockObject(rule), + }; +}; + +const extractDiffableCustomQueryFields = ( + rule: QueryRule | QueryRuleCreateProps +): DiffableCustomQueryFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + alert_suppression: rule.alert_suppression, + }; +}; + +const extractDiffableSavedQueryFieldsFromRuleObject = ( + rule: SavedQueryRule | SavedQueryRuleCreateProps +): DiffableSavedQueryFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + alert_suppression: rule.alert_suppression, + }; +}; + +const extractDiffableEqlFieldsFromRuleObject = ( + rule: EqlRule | EqlRuleCreateProps +): DiffableEqlFields => { + return { + type: rule.type, + data_query: extractRuleEqlQuery(rule.query, rule.language, rule.filters), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + event_category_override: rule.event_category_override, + timestamp_field: rule.timestamp_field, + tiebreaker_field: rule.tiebreaker_field, + }; +}; + +const extractDiffableThreatMatchFieldsFromRuleObject = ( + rule: ThreatMatchRule | ThreatMatchRuleCreateProps +): DiffableThreatMatchFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + threat_query: extractInlineKqlQuery( + rule.threat_query, + rule.threat_language, + rule.threat_filters + ), + threat_index: rule.threat_index, + threat_mapping: rule.threat_mapping, + threat_indicator_path: rule.threat_indicator_path, + concurrent_searches: rule.concurrent_searches, + items_per_search: rule.items_per_search, + }; +}; + +const extractDiffableThresholdFieldsFromRuleObject = ( + rule: ThresholdRule | ThresholdRuleCreateProps +): DiffableThresholdFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + threshold: rule.threshold, + }; +}; + +const extractDiffableMachineLearningFieldsFromRuleObject = ( + rule: MachineLearningRule | MachineLearningRuleCreateProps +): DiffableMachineLearningFields => { + return { + type: rule.type, + machine_learning_job_id: rule.machine_learning_job_id, + anomaly_threshold: rule.anomaly_threshold, + }; +}; + +const extractDiffableNewTermsFieldsFromRuleObject = ( + rule: NewTermsRule | NewTermsRuleCreateProps +): DiffableNewTermsFields => { + return { + type: rule.type, + data_query: extractInlineKqlQuery(rule.query, rule.language, rule.filters), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + new_terms_fields: rule.new_terms_fields, + history_window_start: rule.history_window_start, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_building_block_object.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_building_block_object.ts new file mode 100644 index 0000000000000..fda8efafea6b5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_building_block_object.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../../../../common/detection_engine/rule_schema'; +import type { BuildingBlockObject } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; +import type { PrebuiltRuleAsset } from '../../../model/rule_assets/prebuilt_rule_asset'; + +export const extractBuildingBlockObject = ( + rule: RuleResponse | PrebuiltRuleAsset +): BuildingBlockObject | undefined => { + if (rule.building_block_type == null) { + return undefined; + } + return { + type: rule.building_block_type, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_query.ts new file mode 100644 index 0000000000000..3672dddca4e64 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_query.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + EqlQueryLanguage, + KqlQueryLanguage, + RuleFilterArray, + RuleQuery, +} from '../../../../../../../common/detection_engine/rule_schema'; +import type { + InlineKqlQuery, + RuleEqlQuery, + RuleKqlQuery, +} from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; +import { KqlQueryType } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; + +export const extractRuleKqlQuery = ( + query: RuleQuery | undefined, + language: KqlQueryLanguage | undefined, + filters: RuleFilterArray | undefined, + savedQueryId: string | undefined +): RuleKqlQuery => { + if (savedQueryId != null) { + return { + type: KqlQueryType.saved_query, + saved_query_id: savedQueryId, + }; + } else { + return extractInlineKqlQuery(query, language, filters); + } +}; + +export const extractInlineKqlQuery = ( + query: RuleQuery | undefined, + language: KqlQueryLanguage | undefined, + filters: RuleFilterArray | undefined +): InlineKqlQuery => { + return { + type: KqlQueryType.inline_query, + query: query ?? '', + language: language ?? 'kuery', + filters: filters ?? [], + }; +}; + +export const extractRuleEqlQuery = ( + query: RuleQuery, + language: EqlQueryLanguage, + filters: RuleFilterArray | undefined +): RuleEqlQuery => { + return { + query, + language, + filters: filters ?? [], + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_source.ts new file mode 100644 index 0000000000000..3391407e856dc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_data_source.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + DataViewId, + IndexPatternArray, +} from '../../../../../../../common/detection_engine/rule_schema'; +import type { RuleDataSource } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; +import { DataSourceType } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; + +export const extractRuleDataSource = ( + indexPatterns: IndexPatternArray | undefined, + dataViewId: DataViewId | undefined +): RuleDataSource | undefined => { + if (indexPatterns != null) { + return { + type: DataSourceType.index_patterns, + index_patterns: indexPatterns, + }; + } + + if (dataViewId != null) { + return { + type: DataSourceType.data_view, + data_view_id: dataViewId, + }; + } + + return undefined; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_name_override_object.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_name_override_object.ts new file mode 100644 index 0000000000000..e1e95c12cc9c0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_name_override_object.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../../../../common/detection_engine/rule_schema'; +import type { RuleNameOverrideObject } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; +import type { PrebuiltRuleAsset } from '../../../model/rule_assets/prebuilt_rule_asset'; + +export const extractRuleNameOverrideObject = ( + rule: RuleResponse | PrebuiltRuleAsset +): RuleNameOverrideObject | undefined => { + if (rule.rule_name_override == null) { + return undefined; + } + return { + field_name: rule.rule_name_override, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_schedule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_schedule.ts new file mode 100644 index 0000000000000..21d81d218680c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_rule_schedule.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import dateMath from '@elastic/datemath'; +import { parseDuration } from '@kbn/alerting-plugin/common'; + +import type { RuleResponse } from '../../../../../../../common/detection_engine/rule_schema'; +import type { RuleSchedule } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; +import type { PrebuiltRuleAsset } from '../../../model/rule_assets/prebuilt_rule_asset'; + +export const extractRuleSchedule = (rule: RuleResponse | PrebuiltRuleAsset): RuleSchedule => { + const interval = rule.interval ?? '5m'; + const from = rule.from ?? 'now-6m'; + const to = rule.to ?? 'now'; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ruleMeta = (rule.meta ?? {}) as any; + const lookbackFromMeta = String(ruleMeta.from ?? ''); + + const intervalDuration = parseInterval(interval); + const lookbackFromMetaDuration = parseInterval(lookbackFromMeta); + const driftToleranceDuration = parseDriftTolerance(from, to); + + if (lookbackFromMetaDuration != null) { + if (intervalDuration != null) { + return { + interval, + lookback: lookbackFromMeta, + }; + } + return { + interval: `Cannot parse: interval="${interval}"`, + lookback: lookbackFromMeta, + }; + } + + if (intervalDuration == null) { + return { + interval: `Cannot parse: interval="${interval}"`, + lookback: `Cannot calculate due to invalid interval`, + }; + } + + if (driftToleranceDuration == null) { + return { + interval, + lookback: `Cannot parse: from="${from}", to="${to}"`, + }; + } + + const lookbackDuration = moment.duration().add(driftToleranceDuration).subtract(intervalDuration); + const lookback = `${lookbackDuration.asSeconds()}s`; + + return { interval, lookback }; +}; + +const parseInterval = (intervalString: string): moment.Duration | null => { + try { + const milliseconds = parseDuration(intervalString); + return moment.duration(milliseconds); + } catch (e) { + return null; + } +}; + +const parseDriftTolerance = (from: string, to: string): moment.Duration | null => { + const now = new Date(); + const fromDate = parseDateMathString(from, now); + const toDate = parseDateMathString(to, now); + + if (fromDate == null || toDate == null) { + return null; + } + + return moment.duration(toDate.diff(fromDate)); +}; + +const parseDateMathString = (dateMathString: string, now: Date): moment.Moment | null => { + const parsedDate = dateMath.parse(dateMathString, { forceNow: now }); + return parsedDate != null && parsedDate.isValid() ? parsedDate : null; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timeline_template_reference.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timeline_template_reference.ts new file mode 100644 index 0000000000000..f213f8a5a2189 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timeline_template_reference.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../../../../common/detection_engine/rule_schema'; +import type { TimelineTemplateReference } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; +import type { PrebuiltRuleAsset } from '../../../model/rule_assets/prebuilt_rule_asset'; + +export const extractTimelineTemplateReference = ( + rule: RuleResponse | PrebuiltRuleAsset +): TimelineTemplateReference | undefined => { + if (rule.timeline_id == null) { + return undefined; + } + return { + timeline_id: rule.timeline_id, + timeline_title: rule.timeline_title ?? '', + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timestamp_override_object.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timestamp_override_object.ts new file mode 100644 index 0000000000000..b311557b7d260 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/extract_timestamp_override_object.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../../../../common/detection_engine/rule_schema'; +import type { TimestampOverrideObject } from '../../../../../../../common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types'; +import type { PrebuiltRuleAsset } from '../../../model/rule_assets/prebuilt_rule_asset'; + +export const extractTimestampOverrideObject = ( + rule: RuleResponse | PrebuiltRuleAsset +): TimestampOverrideObject | undefined => { + if (rule.timestamp_override == null) { + return undefined; + } + return { + field_name: rule.timestamp_override, + fallback_disabled: rule.timestamp_override_fallback_disabled ?? false, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_latest_prebuilt_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_latest_prebuilt_rules.ts deleted file mode 100644 index 36fb03c9bac7a..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_latest_prebuilt_rules.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { BadRequestError } from '@kbn/securitysolution-es-utils'; -import { exactCheck, formatErrors } from '@kbn/securitysolution-io-ts-utils'; -import { getOrElse } from 'fp-ts/lib/Either'; -import type * as t from 'io-ts'; -import { PrebuiltRuleToInstall } from '../../../../../common/detection_engine/prebuilt_rules'; -import { withSecuritySpan } from '../../../../utils/with_security_span'; -import type { - IRuleAssetSOAttributes, - IRuleAssetsClient, -} from './rule_asset/rule_asset_saved_objects_client'; - -import { prebuiltRulesToMap } from './utils'; - -export const getLatestPrebuiltRules = async ( - ruleAssetsClient: IRuleAssetsClient -): Promise> => - withSecuritySpan('getLatestPrebuiltRules', async () => { - const ruleAssets = await ruleAssetsClient.fetchLatestVersions(); - return prebuiltRulesToMap(validateRuleAssets(ruleAssets)); - }); - -/** - * Validate the rules from Saved Objects created by Fleet. - */ -const validateRuleAssets = (rules: IRuleAssetSOAttributes[]): PrebuiltRuleToInstall[] => { - return rules.map((rule) => { - const decoded = PrebuiltRuleToInstall.decode(rule); - const checked = exactCheck(rule, decoded); - - const onLeft = (errors: t.Errors): PrebuiltRuleToInstall => { - const ruleName = rule.name ? rule.name : '(rule name unknown)'; - const ruleId = rule.rule_id ? rule.rule_id : '(rule rule_id unknown)'; - throw new BadRequestError( - `name: "${ruleName}", rule_id: "${ruleId}" within the security-rule saved object ` + - `is not a valid detection engine rule. Expect the system ` + - `to not work with pre-packaged rules until this rule is fixed ` + - `or the file is removed. Error is: ${formatErrors( - errors - ).join()}, Full rule contents are:\n${JSON.stringify(rule, null, 2)}` - ); - }; - - return getOrElse(onLeft)(checked); - }); -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.test.ts index b93ef2cc5178f..fc14ab92bbcbb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.test.ts @@ -7,74 +7,65 @@ import { getRulesToInstall } from './get_rules_to_install'; import { getRuleMock } from '../../routes/__mocks__/request_responses'; -import { getPrebuiltRuleMock } from '../../../../../common/detection_engine/prebuilt_rules/mocks'; +import { getPrebuiltRuleMock } from '../mocks'; import { getQueryRuleParams } from '../../rule_schema/mocks'; -import { prebuiltRulesToMap, rulesToMap } from './utils'; +import { rulesToMap } from './utils'; describe('get_rules_to_install', () => { test('should return empty array if both rule sets are empty', () => { - const update = getRulesToInstall(prebuiltRulesToMap([]), rulesToMap([])); + const update = getRulesToInstall([], rulesToMap([])); expect(update).toEqual([]); }); test('should return empty array if the two rule ids match', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-1'; - const update = getRulesToInstall( - prebuiltRulesToMap([ruleFromFileSystem]), - rulesToMap([installedRule]) - ); + const update = getRulesToInstall([ruleAsset], rulesToMap([installedRule])); expect(update).toEqual([]); }); test('should return the rule to install if the id of the two rules do not match', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-2'; - const update = getRulesToInstall( - prebuiltRulesToMap([ruleFromFileSystem]), - rulesToMap([installedRule]) - ); - expect(update).toEqual([ruleFromFileSystem]); + const update = getRulesToInstall([ruleAsset], rulesToMap([installedRule])); + expect(update).toEqual([ruleAsset]); }); test('should return two rules to install if both the ids of the two rules do not match', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.rule_id = 'rule-1'; + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.rule_id = 'rule-1'; - const ruleFromFileSystem2 = getPrebuiltRuleMock(); - ruleFromFileSystem2.rule_id = 'rule-2'; + const ruleAsset2 = getPrebuiltRuleMock(); + ruleAsset2.rule_id = 'rule-2'; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-3'; - const update = getRulesToInstall( - prebuiltRulesToMap([ruleFromFileSystem1, ruleFromFileSystem2]), - rulesToMap([installedRule]) - ); - expect(update).toEqual([ruleFromFileSystem1, ruleFromFileSystem2]); + const update = getRulesToInstall([ruleAsset1, ruleAsset2], rulesToMap([installedRule])); + expect(update).toEqual([ruleAsset1, ruleAsset2]); }); test('should return two rules of three to install if both the ids of the two rules do not match but the third does', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.rule_id = 'rule-1'; + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.rule_id = 'rule-1'; - const ruleFromFileSystem2 = getPrebuiltRuleMock(); - ruleFromFileSystem2.rule_id = 'rule-2'; + const ruleAsset2 = getPrebuiltRuleMock(); + ruleAsset2.rule_id = 'rule-2'; - const ruleFromFileSystem3 = getPrebuiltRuleMock(); - ruleFromFileSystem3.rule_id = 'rule-3'; + const ruleAsset3 = getPrebuiltRuleMock(); + ruleAsset3.rule_id = 'rule-3'; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-3'; const update = getRulesToInstall( - prebuiltRulesToMap([ruleFromFileSystem1, ruleFromFileSystem2, ruleFromFileSystem3]), + [ruleAsset1, ruleAsset2, ruleAsset3], rulesToMap([installedRule]) ); - expect(update).toEqual([ruleFromFileSystem1, ruleFromFileSystem2]); + expect(update).toEqual([ruleAsset1, ruleAsset2]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.ts index 69d76cdc98e4a..d2e6bc4acf7b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_install.ts @@ -5,14 +5,12 @@ * 2.0. */ -import type { PrebuiltRuleToInstall } from '../../../../../common/detection_engine/prebuilt_rules'; import type { RuleAlertType } from '../../rule_schema'; +import type { PrebuiltRuleAsset } from '../model/rule_assets/prebuilt_rule_asset'; export const getRulesToInstall = ( - latestPrebuiltRules: Map, + latestPrebuiltRules: PrebuiltRuleAsset[], installedRules: Map ) => { - return Array.from(latestPrebuiltRules.values()).filter( - (rule) => !installedRules.has(rule.rule_id) - ); + return latestPrebuiltRules.filter((rule) => !installedRules.has(rule.rule_id)); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.test.ts index bcf85d4095556..89d59ec8f77e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.test.ts @@ -7,82 +7,70 @@ import { filterInstalledRules, getRulesToUpdate, mergeExceptionLists } from './get_rules_to_update'; import { getRuleMock } from '../../routes/__mocks__/request_responses'; -import { getPrebuiltRuleMock } from '../../../../../common/detection_engine/prebuilt_rules/mocks'; +import { getPrebuiltRuleMock } from '../mocks'; import { getQueryRuleParams } from '../../rule_schema/mocks'; -import { prebuiltRulesToMap, rulesToMap } from './utils'; +import { rulesToMap } from './utils'; describe('get_rules_to_update', () => { test('should return empty array if both rule sets are empty', () => { - const update = getRulesToUpdate(prebuiltRulesToMap([]), rulesToMap([])); + const update = getRulesToUpdate([], rulesToMap([])); expect(update).toEqual([]); }); test('should return empty array if the rule_id of the two rules do not match', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 2; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 2; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-2'; installedRule.params.version = 1; - const update = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem]), - rulesToMap([installedRule]) - ); + const update = getRulesToUpdate([ruleAsset], rulesToMap([installedRule])); expect(update).toEqual([]); }); test('should return empty array if the version of file system rule is less than the installed version', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 1; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 1; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-1'; installedRule.params.version = 2; - const update = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem]), - rulesToMap([installedRule]) - ); + const update = getRulesToUpdate([ruleAsset], rulesToMap([installedRule])); expect(update).toEqual([]); }); test('should return empty array if the version of file system rule is the same as the installed version', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 1; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 1; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-1'; installedRule.params.version = 1; - const update = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem]), - rulesToMap([installedRule]) - ); + const update = getRulesToUpdate([ruleAsset], rulesToMap([installedRule])); expect(update).toEqual([]); }); test('should return the rule to update if the version of file system rule is greater than the installed version', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 2; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 2; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-1'; installedRule.params.version = 1; installedRule.params.exceptionsList = []; - const update = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem]), - rulesToMap([installedRule]) - ); - expect(update).toEqual([ruleFromFileSystem]); + const update = getRulesToUpdate([ruleAsset], rulesToMap([installedRule])); + expect(update).toEqual([ruleAsset]); }); test('should return 1 rule out of 2 to update if the version of file system rule is greater than the installed version of just one', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 2; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -94,21 +82,18 @@ describe('get_rules_to_update', () => { installedRule2.params.version = 1; installedRule2.params.exceptionsList = []; - const update = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem]), - rulesToMap([installedRule1, installedRule2]) - ); - expect(update).toEqual([ruleFromFileSystem]); + const update = getRulesToUpdate([ruleAsset], rulesToMap([installedRule1, installedRule2])); + expect(update).toEqual([ruleAsset]); }); test('should return 2 rules out of 2 to update if the version of file system rule is greater than the installed version of both', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; - const ruleFromFileSystem2 = getPrebuiltRuleMock(); - ruleFromFileSystem2.rule_id = 'rule-2'; - ruleFromFileSystem2.version = 2; + const ruleAsset2 = getPrebuiltRuleMock(); + ruleAsset2.rule_id = 'rule-2'; + ruleAsset2.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -121,15 +106,15 @@ describe('get_rules_to_update', () => { installedRule2.params.exceptionsList = []; const update = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem1, ruleFromFileSystem2]), + [ruleAsset1, ruleAsset2], rulesToMap([installedRule1, installedRule2]) ); - expect(update).toEqual([ruleFromFileSystem1, ruleFromFileSystem2]); + expect(update).toEqual([ruleAsset1, ruleAsset2]); }); test('should add back an exception_list if it was removed by the end user on an immutable rule during an upgrade', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = [ + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = [ { id: 'endpoint_list', list_id: 'endpoint_list', @@ -137,24 +122,21 @@ describe('get_rules_to_update', () => { type: 'endpoint', }, ]; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; installedRule1.params.version = 1; installedRule1.params.exceptionsList = []; - const [update] = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem1]), - rulesToMap([installedRule1]) - ); - expect(update.exceptions_list).toEqual(ruleFromFileSystem1.exceptions_list); + const [update] = getRulesToUpdate([ruleAsset1], rulesToMap([installedRule1])); + expect(update.exceptions_list).toEqual(ruleAsset1.exceptions_list); }); test('should not remove an additional exception_list if an additional one was added by the end user on an immutable rule during an upgrade', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = [ + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = [ { id: 'endpoint_list', list_id: 'endpoint_list', @@ -162,8 +144,8 @@ describe('get_rules_to_update', () => { type: 'endpoint', }, ]; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -177,19 +159,16 @@ describe('get_rules_to_update', () => { }, ]; - const [update] = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem1]), - rulesToMap([installedRule1]) - ); + const [update] = getRulesToUpdate([ruleAsset1], rulesToMap([installedRule1])); expect(update.exceptions_list).toEqual([ - ...ruleFromFileSystem1.exceptions_list, + ...ruleAsset1.exceptions_list, ...installedRule1.params.exceptionsList, ]); }); test('should not remove an existing exception_list if they are the same between the current installed one and the upgraded one', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = [ + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = [ { id: 'endpoint_list', list_id: 'endpoint_list', @@ -197,8 +176,8 @@ describe('get_rules_to_update', () => { type: 'endpoint', }, ]; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -212,18 +191,15 @@ describe('get_rules_to_update', () => { }, ]; - const [update] = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem1]), - rulesToMap([installedRule1]) - ); - expect(update.exceptions_list).toEqual(ruleFromFileSystem1.exceptions_list); + const [update] = getRulesToUpdate([ruleAsset1], rulesToMap([installedRule1])); + expect(update.exceptions_list).toEqual(ruleAsset1.exceptions_list); }); test('should not remove an existing exception_list if the rule has an empty exceptions list', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = []; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = []; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -237,23 +213,20 @@ describe('get_rules_to_update', () => { }, ]; - const [update] = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem1]), - rulesToMap([installedRule1]) - ); + const [update] = getRulesToUpdate([ruleAsset1], rulesToMap([installedRule1])); expect(update.exceptions_list).toEqual(installedRule1.params.exceptionsList); }); test('should not remove an existing exception_list if the rule has an empty exceptions list for multiple rules', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = []; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = []; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; - const ruleFromFileSystem2 = getPrebuiltRuleMock(); - ruleFromFileSystem2.exceptions_list = []; - ruleFromFileSystem2.rule_id = 'rule-2'; - ruleFromFileSystem2.version = 2; + const ruleAsset2 = getPrebuiltRuleMock(); + ruleAsset2.exceptions_list = []; + ruleAsset2.rule_id = 'rule-2'; + ruleAsset2.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -279,7 +252,7 @@ describe('get_rules_to_update', () => { ]; const [update1, update2] = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem1, ruleFromFileSystem2]), + [ruleAsset1, ruleAsset2], rulesToMap([installedRule1, installedRule2]) ); expect(update1.exceptions_list).toEqual(installedRule1.params.exceptionsList); @@ -287,16 +260,16 @@ describe('get_rules_to_update', () => { }); test('should not remove an existing exception_list if the rule has an empty exceptions list for mixed rules', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = []; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; - - const ruleFromFileSystem2 = getPrebuiltRuleMock(); - ruleFromFileSystem2.exceptions_list = []; - ruleFromFileSystem2.rule_id = 'rule-2'; - ruleFromFileSystem2.version = 2; - ruleFromFileSystem2.exceptions_list = [ + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = []; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; + + const ruleAsset2 = getPrebuiltRuleMock(); + ruleAsset2.exceptions_list = []; + ruleAsset2.rule_id = 'rule-2'; + ruleAsset2.version = 2; + ruleAsset2.exceptions_list = [ { id: 'second_list', list_id: 'second_list', @@ -330,12 +303,12 @@ describe('get_rules_to_update', () => { ]; const [update1, update2] = getRulesToUpdate( - prebuiltRulesToMap([ruleFromFileSystem1, ruleFromFileSystem2]), + [ruleAsset1, ruleAsset2], rulesToMap([installedRule1, installedRule2]) ); expect(update1.exceptions_list).toEqual(installedRule1.params.exceptionsList); expect(update2.exceptions_list).toEqual([ - ...ruleFromFileSystem2.exceptions_list, + ...ruleAsset2.exceptions_list, ...installedRule2.params.exceptionsList, ]); }); @@ -343,60 +316,60 @@ describe('get_rules_to_update', () => { describe('filterInstalledRules', () => { test('should return "false" if the id of the two rules do not match', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 2; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 2; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-2'; installedRule.params.version = 1; - const shouldUpdate = filterInstalledRules(ruleFromFileSystem, rulesToMap([installedRule])); + const shouldUpdate = filterInstalledRules(ruleAsset, rulesToMap([installedRule])); expect(shouldUpdate).toEqual(false); }); test('should return "false" if the version of file system rule is less than the installed version', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 1; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 1; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-1'; installedRule.params.version = 2; - const shouldUpdate = filterInstalledRules(ruleFromFileSystem, rulesToMap([installedRule])); + const shouldUpdate = filterInstalledRules(ruleAsset, rulesToMap([installedRule])); expect(shouldUpdate).toEqual(false); }); test('should return "false" if the version of file system rule is the same as the installed version', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 1; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 1; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-1'; installedRule.params.version = 1; - const shouldUpdate = filterInstalledRules(ruleFromFileSystem, rulesToMap([installedRule])); + const shouldUpdate = filterInstalledRules(ruleAsset, rulesToMap([installedRule])); expect(shouldUpdate).toEqual(false); }); test('should return "true" to update if the version of file system rule is greater than the installed version', () => { - const ruleFromFileSystem = getPrebuiltRuleMock(); - ruleFromFileSystem.rule_id = 'rule-1'; - ruleFromFileSystem.version = 2; + const ruleAsset = getPrebuiltRuleMock(); + ruleAsset.rule_id = 'rule-1'; + ruleAsset.version = 2; const installedRule = getRuleMock(getQueryRuleParams()); installedRule.params.ruleId = 'rule-1'; installedRule.params.version = 1; installedRule.params.exceptionsList = []; - const shouldUpdate = filterInstalledRules(ruleFromFileSystem, rulesToMap([installedRule])); + const shouldUpdate = filterInstalledRules(ruleAsset, rulesToMap([installedRule])); expect(shouldUpdate).toEqual(true); }); }); describe('mergeExceptionLists', () => { test('should add back an exception_list if it was removed by the end user on an immutable rule during an upgrade', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = [ + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = [ { id: 'endpoint_list', list_id: 'endpoint_list', @@ -404,21 +377,21 @@ describe('mergeExceptionLists', () => { type: 'endpoint', }, ]; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; installedRule1.params.version = 1; installedRule1.params.exceptionsList = []; - const update = mergeExceptionLists(ruleFromFileSystem1, rulesToMap([installedRule1])); - expect(update.exceptions_list).toEqual(ruleFromFileSystem1.exceptions_list); + const update = mergeExceptionLists(ruleAsset1, rulesToMap([installedRule1])); + expect(update.exceptions_list).toEqual(ruleAsset1.exceptions_list); }); test('should not remove an additional exception_list if an additional one was added by the end user on an immutable rule during an upgrade', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = [ + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = [ { id: 'endpoint_list', list_id: 'endpoint_list', @@ -426,8 +399,8 @@ describe('mergeExceptionLists', () => { type: 'endpoint', }, ]; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -441,16 +414,16 @@ describe('mergeExceptionLists', () => { }, ]; - const update = mergeExceptionLists(ruleFromFileSystem1, rulesToMap([installedRule1])); + const update = mergeExceptionLists(ruleAsset1, rulesToMap([installedRule1])); expect(update.exceptions_list).toEqual([ - ...ruleFromFileSystem1.exceptions_list, + ...ruleAsset1.exceptions_list, ...installedRule1.params.exceptionsList, ]); }); test('should not remove an existing exception_list if they are the same between the current installed one and the upgraded one', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = [ + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = [ { id: 'endpoint_list', list_id: 'endpoint_list', @@ -458,8 +431,8 @@ describe('mergeExceptionLists', () => { type: 'endpoint', }, ]; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -473,15 +446,15 @@ describe('mergeExceptionLists', () => { }, ]; - const update = mergeExceptionLists(ruleFromFileSystem1, rulesToMap([installedRule1])); - expect(update.exceptions_list).toEqual(ruleFromFileSystem1.exceptions_list); + const update = mergeExceptionLists(ruleAsset1, rulesToMap([installedRule1])); + expect(update.exceptions_list).toEqual(ruleAsset1.exceptions_list); }); test('should not remove an existing exception_list if the rule has an empty exceptions list', () => { - const ruleFromFileSystem1 = getPrebuiltRuleMock(); - ruleFromFileSystem1.exceptions_list = []; - ruleFromFileSystem1.rule_id = 'rule-1'; - ruleFromFileSystem1.version = 2; + const ruleAsset1 = getPrebuiltRuleMock(); + ruleAsset1.exceptions_list = []; + ruleAsset1.rule_id = 'rule-1'; + ruleAsset1.version = 2; const installedRule1 = getRuleMock(getQueryRuleParams()); installedRule1.params.ruleId = 'rule-1'; @@ -495,7 +468,7 @@ describe('mergeExceptionLists', () => { }, ]; - const update = mergeExceptionLists(ruleFromFileSystem1, rulesToMap([installedRule1])); + const update = mergeExceptionLists(ruleAsset1, rulesToMap([installedRule1])); expect(update.exceptions_list).toEqual(installedRule1.params.exceptionsList); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.ts index 7547f16961cc1..e25ca4f1348e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/get_rules_to_update.ts @@ -5,8 +5,8 @@ * 2.0. */ -import type { PrebuiltRuleToInstall } from '../../../../../common/detection_engine/prebuilt_rules'; import type { RuleAlertType } from '../../rule_schema'; +import type { PrebuiltRuleAsset } from '../model/rule_assets/prebuilt_rule_asset'; /** * Returns the rules to update by doing a compare to the rules from the file system against @@ -16,10 +16,10 @@ import type { RuleAlertType } from '../../rule_schema'; * @param installedRules The installed rules */ export const getRulesToUpdate = ( - latestPrebuiltRules: Map, + latestPrebuiltRules: PrebuiltRuleAsset[], installedRules: Map ) => { - return Array.from(latestPrebuiltRules.values()) + return latestPrebuiltRules .filter((latestRule) => filterInstalledRules(latestRule, installedRules)) .map((latestRule) => mergeExceptionLists(latestRule, installedRules)); }; @@ -31,7 +31,7 @@ export const getRulesToUpdate = ( * @param installedRules The installed rules to compare against for updates */ export const filterInstalledRules = ( - latestPrebuiltRule: PrebuiltRuleToInstall, + latestPrebuiltRule: PrebuiltRuleAsset, installedRules: Map ): boolean => { const installedRule = installedRules.get(latestPrebuiltRule.rule_id); @@ -46,9 +46,9 @@ export const filterInstalledRules = ( * @param installedRules The installed rules which might have user driven exceptions_lists */ export const mergeExceptionLists = ( - latestPrebuiltRule: PrebuiltRuleToInstall, + latestPrebuiltRule: PrebuiltRuleAsset, installedRules: Map -): PrebuiltRuleToInstall => { +): PrebuiltRuleAsset => { if (latestPrebuiltRule.exceptions_list != null) { const installedRule = installedRules.get(latestPrebuiltRule.rule_id); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_objects_client.ts deleted file mode 100644 index 556ebbe2df108..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_objects_client.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { AggregationsMultiBucketAggregateBase } from '@elastic/elasticsearch/lib/api/types'; -import type { AggregationsTopHitsAggregate } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { SavedObjectsClientContract } from '@kbn/core/server'; -import { invariant } from '../../../../../../common/utils/invariant'; -import { withSecuritySpan } from '../../../../../utils/with_security_span'; -import { ruleAssetSavedObjectType } from './rule_asset_saved_object_mappings'; - -const MAX_PREBUILT_RULES_COUNT = 10_000; - -export interface IRuleAssetSOAttributes extends Record { - rule_id: string | null | undefined; - version: string | null | undefined; - name: string | null | undefined; -} - -export interface IRuleAssetSavedObject { - type: string; - id: string; - attributes: IRuleAssetSOAttributes; -} - -export interface IRuleAssetsClient { - fetchLatestVersions: () => Promise; -} - -export const ruleAssetsClientFactory = ( - savedObjectsClient: SavedObjectsClientContract -): IRuleAssetsClient => { - return { - fetchLatestVersions: () => { - return withSecuritySpan('RuleAssetsClient.fetchLatestVersions', async () => { - const findResult = await savedObjectsClient.find< - IRuleAssetSavedObject, - { - rules: AggregationsMultiBucketAggregateBase<{ - latest_version: AggregationsTopHitsAggregate; - }>; - } - >({ - type: ruleAssetSavedObjectType, - aggs: { - rules: { - terms: { - field: `${ruleAssetSavedObjectType}.attributes.rule_id`, - size: MAX_PREBUILT_RULES_COUNT, - }, - aggs: { - latest_version: { - top_hits: { - size: 1, - sort: { - [`${ruleAssetSavedObjectType}.version`]: 'desc', - }, - }, - }, - }, - }, - }, - }); - const buckets = findResult.aggregations?.rules?.buckets ?? []; - - invariant(Array.isArray(buckets), 'Expected buckets to be an array'); - - return buckets.map((bucket) => { - const hit = bucket.latest_version.hits.hits[0]; - return hit._source[ruleAssetSavedObjectType]; - }); - }); - }, - }; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts new file mode 100644 index 0000000000000..96cb684a152d9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { chunk } from 'lodash'; +import type { AggregationsMultiBucketAggregateBase } from '@elastic/elasticsearch/lib/api/types'; +import type { AggregationsTopHitsAggregate } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { SavedObjectsBulkCreateObject, SavedObjectsClientContract } from '@kbn/core/server'; +import { invariant } from '../../../../../../common/utils/invariant'; +import { withSecuritySpan } from '../../../../../utils/with_security_span'; +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; +import { validatePrebuiltRuleAssets } from './prebuilt_rule_assets_validation'; +import { PREBUILT_RULE_ASSETS_SO_TYPE } from './prebuilt_rule_assets_type'; +import type { PrebuiltRuleVersionInfo } from '../../model/rule_versions/prebuilt_rule_version_info'; + +const MAX_PREBUILT_RULES_COUNT = 10_000; +const MAX_ASSETS_PER_BULK_CREATE_REQUEST = 500; + +export interface IPrebuiltRuleAssetsClient { + fetchLatestAssets: () => Promise; + + fetchLatestVersions(): Promise; + + fetchAssetsByVersionInfo(versions: PrebuiltRuleVersionInfo[]): Promise; + + bulkCreateAssets(assets: PrebuiltRuleAsset[]): Promise; +} + +export const createPrebuiltRuleAssetsClient = ( + savedObjectsClient: SavedObjectsClientContract +): IPrebuiltRuleAssetsClient => { + return { + fetchLatestAssets: () => { + return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestAssets', async () => { + const findResult = await savedObjectsClient.find< + PrebuiltRuleAsset, + { + rules: AggregationsMultiBucketAggregateBase<{ + latest_version: AggregationsTopHitsAggregate; + }>; + } + >({ + type: PREBUILT_RULE_ASSETS_SO_TYPE, + aggs: { + rules: { + terms: { + field: `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id`, + size: MAX_PREBUILT_RULES_COUNT, + }, + aggs: { + latest_version: { + top_hits: { + size: 1, + sort: { + [`${PREBUILT_RULE_ASSETS_SO_TYPE}.version`]: 'desc', + }, + }, + }, + }, + }, + }, + }); + + const buckets = findResult.aggregations?.rules?.buckets ?? []; + invariant(Array.isArray(buckets), 'Expected buckets to be an array'); + + const ruleAssets = buckets.map((bucket) => { + const hit = bucket.latest_version.hits.hits[0]; + return hit._source[PREBUILT_RULE_ASSETS_SO_TYPE]; + }); + + return validatePrebuiltRuleAssets(ruleAssets); + }); + }, + + fetchLatestVersions: (): Promise => { + return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestVersions', async () => { + const findResult = await savedObjectsClient.find< + PrebuiltRuleAsset, + { + rules: AggregationsMultiBucketAggregateBase<{ + latest_version: AggregationsTopHitsAggregate; + }>; + } + >({ + type: PREBUILT_RULE_ASSETS_SO_TYPE, + aggs: { + rules: { + terms: { + field: `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id`, + size: MAX_PREBUILT_RULES_COUNT, + }, + aggs: { + latest_version: { + top_hits: { + size: 1, + sort: [ + { + [`${PREBUILT_RULE_ASSETS_SO_TYPE}.version`]: 'desc', + }, + ], + _source: [ + `${PREBUILT_RULE_ASSETS_SO_TYPE}.rule_id`, + `${PREBUILT_RULE_ASSETS_SO_TYPE}.version`, + ], + }, + }, + }, + }, + }, + }); + + const buckets = findResult.aggregations?.rules?.buckets ?? []; + invariant(Array.isArray(buckets), 'Expected buckets to be an array'); + + return buckets.map((bucket) => { + const hit = bucket.latest_version.hits.hits[0]; + const soAttributes = hit._source[PREBUILT_RULE_ASSETS_SO_TYPE]; + const versionInfo: PrebuiltRuleVersionInfo = { + rule_id: soAttributes.rule_id, + version: soAttributes.version, + }; + return versionInfo; + }); + }); + }, + + fetchAssetsByVersionInfo: ( + versions: PrebuiltRuleVersionInfo[] + ): Promise => { + return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchAssetsByVersionInfo', async () => { + if (versions.length === 0) { + // NOTE: without early return it would build incorrect filter and fetch all existing saved objects + return []; + } + + const attr = `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes`; + const filter = versions + .map((v) => `(${attr}.rule_id: ${v.rule_id} AND ${attr}.version: ${v.version})`) + .join(' OR '); + + const findResult = await savedObjectsClient.find({ + type: PREBUILT_RULE_ASSETS_SO_TYPE, + filter, + perPage: MAX_PREBUILT_RULES_COUNT, + }); + + const ruleAssets = findResult.saved_objects.map((so) => so.attributes); + return validatePrebuiltRuleAssets(ruleAssets); + }); + }, + + bulkCreateAssets: (assets: PrebuiltRuleAsset[]): Promise => { + return withSecuritySpan('IPrebuiltRuleAssetsClient.bulkCreateAssets', async () => { + const validAssets = validatePrebuiltRuleAssets(assets); + const bulkCreateObjects: Array> = + validAssets.map((asset) => ({ + id: `${asset.rule_id}_${asset.version}`, + type: PREBUILT_RULE_ASSETS_SO_TYPE, + attributes: asset, + })); + + const bulkCreateChunks = chunk(bulkCreateObjects, MAX_ASSETS_PER_BULK_CREATE_REQUEST); + + for (const chunkOfObjects of bulkCreateChunks) { + await savedObjectsClient.bulkCreate(chunkOfObjects, { + refresh: false, + overwrite: true, + }); + } + }); + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_type.ts similarity index 70% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_object_mappings.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_type.ts index 71505fac2d4c5..04bdf7ca28fca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_type.ts @@ -7,9 +7,9 @@ import type { SavedObjectsType } from '@kbn/core/server'; -export const ruleAssetSavedObjectType = 'security-rule'; +export const PREBUILT_RULE_ASSETS_SO_TYPE = 'security-rule'; -export const ruleAssetSavedObjectMappings: SavedObjectsType['mappings'] = { +const prebuiltRuleAssetMappings: SavedObjectsType['mappings'] = { dynamic: false, properties: { name: { @@ -24,13 +24,13 @@ export const ruleAssetSavedObjectMappings: SavedObjectsType['mappings'] = { }, }; -export const ruleAssetType: SavedObjectsType = { - name: ruleAssetSavedObjectType, +export const prebuiltRuleAssetType: SavedObjectsType = { + name: PREBUILT_RULE_ASSETS_SO_TYPE, hidden: false, management: { importableAndExportable: true, visibleInManagement: false, }, namespaceType: 'agnostic', - mappings: ruleAssetSavedObjectMappings, + mappings: prebuiltRuleAssetMappings, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_validation.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_validation.ts new file mode 100644 index 0000000000000..6ba2417fc5263 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_validation.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as t from 'io-ts'; +import { getOrElse } from 'fp-ts/lib/Either'; +import { BadRequestError } from '@kbn/securitysolution-es-utils'; +import { exactCheck, formatErrors } from '@kbn/securitysolution-io-ts-utils'; +import { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; + +export const validatePrebuiltRuleAssets = (rules: PrebuiltRuleAsset[]): PrebuiltRuleAsset[] => { + return rules.map((rule) => validatePrebuiltRuleAsset(rule)); +}; + +export const validatePrebuiltRuleAsset = (rule: PrebuiltRuleAsset): PrebuiltRuleAsset => { + const decoded = PrebuiltRuleAsset.decode(rule); + const checked = exactCheck(rule, decoded); + + const onLeft = (errors: t.Errors): PrebuiltRuleAsset => { + const ruleName = rule.name ? rule.name : '(rule name unknown)'; + const ruleId = rule.rule_id ? rule.rule_id : '(rule rule_id unknown)'; + throw new BadRequestError( + `name: "${ruleName}", rule_id: "${ruleId}" within the security-rule saved object ` + + `is not a valid detection engine rule. Expect the system ` + + `to not work with pre-packaged rules until this rule is fixed ` + + `or the file is removed. Error is: ${formatErrors( + errors + ).join()}, Full rule contents are:\n${JSON.stringify(rule, null, 2)}` + ); + }; + + return getOrElse(onLeft)(checked); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/create_prebuilt_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/create_prebuilt_rules.ts similarity index 72% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/create_prebuilt_rules.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/create_prebuilt_rules.ts index 54554e50a7dd7..2fd8ef266e6b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/create_prebuilt_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/create_prebuilt_rules.ts @@ -6,13 +6,13 @@ */ import type { RulesClient } from '@kbn/alerting-plugin/server'; -import { MAX_RULES_TO_UPDATE_IN_PARALLEL } from '../../../../../common/constants'; -import type { PrebuiltRuleToInstall } from '../../../../../common/detection_engine/prebuilt_rules'; -import { initPromisePool } from '../../../../utils/promise_pool'; -import { withSecuritySpan } from '../../../../utils/with_security_span'; -import { createRules } from '../../rule_management/logic/crud/create_rules'; +import { MAX_RULES_TO_UPDATE_IN_PARALLEL } from '../../../../../../common/constants'; +import { initPromisePool } from '../../../../../utils/promise_pool'; +import { withSecuritySpan } from '../../../../../utils/with_security_span'; +import { createRules } from '../../../rule_management/logic/crud/create_rules'; +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; -export const createPrebuiltRules = (rulesClient: RulesClient, rules: PrebuiltRuleToInstall[]) => +export const createPrebuiltRules = (rulesClient: RulesClient, rules: PrebuiltRuleAsset[]) => withSecuritySpan('createPrebuiltRules', async () => { const result = await initPromisePool({ concurrency: MAX_RULES_TO_UPDATE_IN_PARALLEL, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts new file mode 100644 index 0000000000000..f0d978376d330 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { RuleResponse } from '../../../../../../common/detection_engine/rule_schema'; +import { withSecuritySpan } from '../../../../../utils/with_security_span'; +import { getExistingPrepackagedRules } from '../../../rule_management/logic/search/get_existing_prepackaged_rules'; +import { internalRuleToAPIResponse } from '../../../rule_management/normalization/rule_converters'; +import type { PrebuiltRuleVersionInfo } from '../../model/rule_versions/prebuilt_rule_version_info'; + +export interface IPrebuiltRuleObjectsClient { + fetchInstalledRules(): Promise; +} + +export interface FetchInstalledRulesResult { + installedRules: RuleResponse[]; + installedVersions: PrebuiltRuleVersionInfo[]; +} + +export const createPrebuiltRuleObjectsClient = ( + rulesClient: RulesClient +): IPrebuiltRuleObjectsClient => { + return { + fetchInstalledRules: (): Promise => { + return withSecuritySpan('IPrebuiltRuleObjectsClient.fetchInstalledRules', async () => { + const rulesData = await getExistingPrepackagedRules({ rulesClient }); + const rules = rulesData.map((rule) => internalRuleToAPIResponse(rule)); + const versions = rules.map((rule) => convertRuleToVersionInfo(rule)); + return { + installedRules: rules, + installedVersions: versions, + }; + }); + }, + }; +}; + +const convertRuleToVersionInfo = (rule: RuleResponse): PrebuiltRuleVersionInfo => { + const versionInfo: PrebuiltRuleVersionInfo = { + rule_id: rule.rule_id, + version: rule.version, + }; + return versionInfo; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/update_prebuilt_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/update_prebuilt_rules.test.ts similarity index 70% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/update_prebuilt_rules.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/update_prebuilt_rules.test.ts index 955d288068e33..13e91b1bc2dfa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/update_prebuilt_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/update_prebuilt_rules.test.ts @@ -7,22 +7,21 @@ import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { getRuleMock, getFindResultWithSingleHit } from '../../routes/__mocks__/request_responses'; -import { updatePrebuiltRules } from './update_prebuilt_rules'; -import { patchRules } from '../../rule_management/logic/crud/patch_rules'; import { - getPrebuiltRuleMock, - getPrebuiltThreatMatchRuleMock, -} from '../../../../../common/detection_engine/prebuilt_rules/mocks'; -import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; -import { legacyMigrate } from '../../rule_management'; -import { getQueryRuleParams, getThreatRuleParams } from '../../rule_schema/mocks'; + getRuleMock, + getFindResultWithSingleHit, +} from '../../../routes/__mocks__/request_responses'; +import { updatePrebuiltRules } from './update_prebuilt_rules'; +import { patchRules } from '../../../rule_management/logic/crud/patch_rules'; +import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from '../../mocks'; +import { legacyMigrate } from '../../../rule_management'; +import { getQueryRuleParams, getThreatRuleParams } from '../../../rule_schema/mocks'; -jest.mock('../../rule_management/logic/crud/patch_rules'); +jest.mock('../../../rule_management/logic/crud/patch_rules'); -jest.mock('../../rule_management/logic/rule_actions/legacy_action_migration', () => { +jest.mock('../../../rule_management/logic/rule_actions/legacy_action_migration', () => { const actual = jest.requireActual( - '../../rule_management/logic/rule_actions/legacy_action_migration' + '../../../rule_management/logic/rule_actions/legacy_action_migration' ); return { ...actual, @@ -33,12 +32,10 @@ jest.mock('../../rule_management/logic/rule_actions/legacy_action_migration', () describe('updatePrebuiltRules', () => { let rulesClient: ReturnType; let savedObjectsClient: ReturnType; - let ruleExecutionLog: ReturnType; beforeEach(() => { rulesClient = rulesClientMock.create(); savedObjectsClient = savedObjectsClientMock.create(); - ruleExecutionLog = ruleExecutionLogMock.forRoutes.create(); (legacyMigrate as jest.Mock).mockResolvedValue(getRuleMock(getQueryRuleParams())); }); @@ -55,12 +52,7 @@ describe('updatePrebuiltRules', () => { const prepackagedRule = getPrebuiltRuleMock(); rulesClient.find.mockResolvedValue(getFindResultWithSingleHit()); - await updatePrebuiltRules( - rulesClient, - savedObjectsClient, - [{ ...prepackagedRule, actions }], - ruleExecutionLog - ); + await updatePrebuiltRules(rulesClient, savedObjectsClient, [{ ...prepackagedRule, actions }]); expect(patchRules).toHaveBeenCalledWith( expect.objectContaining({ @@ -92,12 +84,9 @@ describe('updatePrebuiltRules', () => { }); (legacyMigrate as jest.Mock).mockResolvedValue(getRuleMock(getThreatRuleParams())); - await updatePrebuiltRules( - rulesClient, - savedObjectsClient, - [{ ...prepackagedRule, ...updatedThreatParams }], - ruleExecutionLog - ); + await updatePrebuiltRules(rulesClient, savedObjectsClient, [ + { ...prepackagedRule, ...updatedThreatParams }, + ]); expect(patchRules).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/update_prebuilt_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/update_prebuilt_rules.ts similarity index 73% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/update_prebuilt_rules.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/update_prebuilt_rules.ts index 4956b9377c947..407ffac13e48c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/update_prebuilt_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/update_prebuilt_rules.ts @@ -9,20 +9,19 @@ import { chunk } from 'lodash/fp'; import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { RulesClient, PartialRule } from '@kbn/alerting-plugin/server'; -import type { PrebuiltRuleToInstall } from '../../../../../common/detection_engine/prebuilt_rules'; -import { transformAlertToRuleAction } from '../../../../../common/detection_engine/transform_actions'; -import { MAX_RULES_TO_UPDATE_IN_PARALLEL } from '../../../../../common/constants'; +import { transformAlertToRuleAction } from '../../../../../../common/detection_engine/transform_actions'; +import { MAX_RULES_TO_UPDATE_IN_PARALLEL } from '../../../../../../common/constants'; -import { legacyMigrate } from '../../rule_management'; -import { createRules } from '../../rule_management/logic/crud/create_rules'; -import { readRules } from '../../rule_management/logic/crud/read_rules'; -import { patchRules } from '../../rule_management/logic/crud/patch_rules'; -import { deleteRules } from '../../rule_management/logic/crud/delete_rules'; +import { legacyMigrate } from '../../../rule_management'; +import { createRules } from '../../../rule_management/logic/crud/create_rules'; +import { readRules } from '../../../rule_management/logic/crud/read_rules'; +import { patchRules } from '../../../rule_management/logic/crud/patch_rules'; +import { deleteRules } from '../../../rule_management/logic/crud/delete_rules'; -import type { IRuleExecutionLogForRoutes } from '../../rule_monitoring'; -import type { RuleParams } from '../../rule_schema'; +import type { RuleParams } from '../../../rule_schema'; -import { PrepackagedRulesError } from '../api/install_prebuilt_rules_and_timelines/route'; +import { PrepackagedRulesError } from '../../api/install_prebuilt_rules_and_timelines/route'; +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; /** * Updates existing prebuilt rules given a set of rules and output index. @@ -35,17 +34,11 @@ import { PrepackagedRulesError } from '../api/install_prebuilt_rules_and_timelin export const updatePrebuiltRules = async ( rulesClient: RulesClient, savedObjectsClient: SavedObjectsClientContract, - rules: PrebuiltRuleToInstall[], - ruleExecutionLog: IRuleExecutionLogForRoutes + rules: PrebuiltRuleAsset[] ): Promise => { const ruleChunks = chunk(MAX_RULES_TO_UPDATE_IN_PARALLEL, rules); for (const ruleChunk of ruleChunks) { - const rulePromises = createPromises( - rulesClient, - savedObjectsClient, - ruleChunk, - ruleExecutionLog - ); + const rulePromises = createPromises(rulesClient, savedObjectsClient, ruleChunk); await Promise.all(rulePromises); } }; @@ -60,8 +53,7 @@ export const updatePrebuiltRules = async ( const createPromises = ( rulesClient: RulesClient, savedObjectsClient: SavedObjectsClientContract, - rules: PrebuiltRuleToInstall[], - ruleExecutionLog: IRuleExecutionLogForRoutes + rules: PrebuiltRuleAsset[] ): Array | null>> => { return rules.map(async (rule) => { const existingRule = await readRules({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/utils.ts index 09f231f51d4ac..9c1b3fdff54b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/utils.ts @@ -5,18 +5,8 @@ * 2.0. */ -import type { PrebuiltRuleToInstall } from '../../../../../common/detection_engine/prebuilt_rules'; import type { RuleAlertType } from '../../rule_schema'; -/** - * Converts an array of prebuilt rules to a Map with rule IDs as keys - * - * @param rules Array of prebuilt rules - * @returns Map - */ -export const prebuiltRulesToMap = (rules: PrebuiltRuleToInstall[]) => - new Map(rules.map((rule) => [rule.rule_id, rule])); - /** * Converts an array of rules to a Map with rule IDs as keys * diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/mocks.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/mocks.ts similarity index 80% rename from x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/mocks.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/mocks.ts index 2850442ce975c..81cc9e7b93bff 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/mocks.ts @@ -5,4 +5,4 @@ * 2.0. */ -export * from './model/prebuilt_rule.mock'; +export * from './model/rule_assets/prebuilt_rule_asset.mock'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.mock.ts similarity index 88% rename from x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.mock.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.mock.ts index 24edacde6f715..8329204637c3b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.mock.ts @@ -5,9 +5,9 @@ * 2.0. */ -import type { PrebuiltRuleToInstall } from './prebuilt_rule'; +import type { PrebuiltRuleAsset } from './prebuilt_rule_asset'; -export const getPrebuiltRuleMock = (): PrebuiltRuleToInstall => ({ +export const getPrebuiltRuleMock = (): PrebuiltRuleAsset => ({ description: 'some description', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', @@ -19,7 +19,7 @@ export const getPrebuiltRuleMock = (): PrebuiltRuleToInstall => ({ version: 1, }); -export const getPrebuiltRuleWithExceptionsMock = (): PrebuiltRuleToInstall => ({ +export const getPrebuiltRuleWithExceptionsMock = (): PrebuiltRuleAsset => ({ description: 'A rule with an exception list', name: 'A rule with an exception list', query: 'user.name: root or user.name: admin', @@ -39,7 +39,7 @@ export const getPrebuiltRuleWithExceptionsMock = (): PrebuiltRuleToInstall => ({ version: 2, }); -export const getPrebuiltThreatMatchRuleMock = (): PrebuiltRuleToInstall => ({ +export const getPrebuiltThreatMatchRuleMock = (): PrebuiltRuleAsset => ({ description: 'some description', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts similarity index 81% rename from x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts index 4d08c9391fab8..7960167a993ef 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts @@ -9,16 +9,16 @@ import { left } from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/pipeable'; import { exactCheck, foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; -import { getListArrayMock } from '../../schemas/types/lists.mock'; +import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; -import { PrebuiltRuleToInstall } from './prebuilt_rule'; -import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from './prebuilt_rule.mock'; +import { PrebuiltRuleAsset } from './prebuilt_rule_asset'; +import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from './prebuilt_rule_asset.mock'; -describe('Prebuilt rule schema', () => { +describe('Prebuilt rule asset schema', () => { test('empty objects do not validate', () => { - const payload: Partial = {}; + const payload: Partial = {}; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toContain( @@ -43,12 +43,12 @@ describe('Prebuilt rule schema', () => { }); test('made up values do not validate', () => { - const payload: PrebuiltRuleToInstall & { madeUp: string } = { + const payload: PrebuiltRuleAsset & { madeUp: string } = { ...getPrebuiltRuleMock(), madeUp: 'hi', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeUp"']); @@ -56,11 +56,11 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toContain( @@ -82,12 +82,12 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toContain( @@ -106,13 +106,13 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toContain( @@ -131,14 +131,14 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', to: 'now', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toContain( @@ -157,7 +157,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, name] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -165,7 +165,7 @@ describe('Prebuilt rule schema', () => { name: 'some-name', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toContain( @@ -181,7 +181,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, name, severity] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -190,7 +190,7 @@ describe('Prebuilt rule schema', () => { severity: 'low', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toContain( @@ -203,7 +203,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, name, severity, type] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -213,7 +213,7 @@ describe('Prebuilt rule schema', () => { type: 'query', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -224,7 +224,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, name, severity, type, interval] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -235,7 +235,7 @@ describe('Prebuilt rule schema', () => { type: 'query', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -246,7 +246,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, name, severity, type, interval, index] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -258,7 +258,7 @@ describe('Prebuilt rule schema', () => { index: ['index-1'], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -269,7 +269,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, name, severity, type, query, index, interval, version] does validate', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -284,14 +284,14 @@ describe('Prebuilt rule schema', () => { version: 1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('[rule_id, description, from, to, index, name, severity, interval, type, query, language] does not validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -306,7 +306,7 @@ describe('Prebuilt rule schema', () => { risk_score: 50, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -316,7 +316,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, version] does validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -332,14 +332,14 @@ describe('Prebuilt rule schema', () => { version: 1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, output_index] does not validate', () => { - const payload: Partial & { output_index: string } = { + const payload: Partial & { output_index: string } = { rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -355,7 +355,7 @@ describe('Prebuilt rule schema', () => { language: 'kuery', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -365,7 +365,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, version] does validate', () => { - const payload: Partial = { + const payload: Partial = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -379,38 +379,38 @@ describe('Prebuilt rule schema', () => { version: 1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You can send in a namespace', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), namespace: 'a namespace', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You can send in an empty array to threat', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), threat: [], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, output_index, threat] does validate', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -441,31 +441,31 @@ describe('Prebuilt rule schema', () => { version: 1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('allows references to be sent as valid', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), references: ['index-1'], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('immutable cannot be set in a pre-packaged rule', () => { - const payload: PrebuiltRuleToInstall & { immutable: boolean } = { + const payload: PrebuiltRuleAsset & { immutable: boolean } = { ...getPrebuiltRuleMock(), immutable: true, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual(['invalid keys "immutable"']); @@ -473,11 +473,11 @@ describe('Prebuilt rule schema', () => { }); test('rule_id is required', () => { - const payload: PrebuiltRuleToInstall = getPrebuiltRuleMock(); + const payload: PrebuiltRuleAsset = getPrebuiltRuleMock(); // @ts-expect-error delete payload.rule_id; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -487,12 +487,12 @@ describe('Prebuilt rule schema', () => { }); test('references cannot be numbers', () => { - const payload: Omit & { references: number[] } = { + const payload: Omit & { references: number[] } = { ...getPrebuiltRuleMock(), references: [5], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual(['Invalid value "5" supplied to "references"']); @@ -500,12 +500,12 @@ describe('Prebuilt rule schema', () => { }); test('indexes cannot be numbers', () => { - const payload: Omit & { index: number[] } = { + const payload: Omit & { index: number[] } = { ...getPrebuiltRuleMock(), index: [5], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual(['Invalid value "5" supplied to "index"']); @@ -518,19 +518,19 @@ describe('Prebuilt rule schema', () => { filters: [], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('filters cannot be a string', () => { - const payload: Omit & { filters: string } = { + const payload: Omit & { filters: string } = { ...getPrebuiltRuleMock(), filters: 'some string', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -545,7 +545,7 @@ describe('Prebuilt rule schema', () => { language: 'kuery', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); @@ -557,19 +557,19 @@ describe('Prebuilt rule schema', () => { language: 'lucene', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('language does not validate with something made up', () => { - const payload: Omit & { language: string } = { + const payload: Omit & { language: string } = { ...getPrebuiltRuleMock(), language: 'something-made-up', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -579,12 +579,12 @@ describe('Prebuilt rule schema', () => { }); test('max_signals cannot be negative', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), max_signals: -1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -594,12 +594,12 @@ describe('Prebuilt rule schema', () => { }); test('max_signals cannot be zero', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), max_signals: 0, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual(['Invalid value "0" supplied to "max_signals"']); @@ -607,36 +607,36 @@ describe('Prebuilt rule schema', () => { }); test('max_signals can be 1', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), max_signals: 1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You can optionally send in an array of tags', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), tags: ['tag_1', 'tag_2'], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You cannot send in an array of tags that are numbers', () => { - const payload: Omit & { tags: number[] } = { + const payload: Omit & { tags: number[] } = { ...getPrebuiltRuleMock(), tags: [0, 1, 2], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -648,8 +648,8 @@ describe('Prebuilt rule schema', () => { }); test('You cannot send in an array of threat that are missing "framework"', () => { - const payload: Omit & { - threat: Array>>; + const payload: Omit & { + threat: Array>>; } = { ...getPrebuiltRuleMock(), threat: [ @@ -670,7 +670,7 @@ describe('Prebuilt rule schema', () => { ], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -680,8 +680,8 @@ describe('Prebuilt rule schema', () => { }); test('You cannot send in an array of threat that are missing "tactic"', () => { - const payload: Omit & { - threat: Array>>; + const payload: Omit & { + threat: Array>>; } = { ...getPrebuiltRuleMock(), threat: [ @@ -698,7 +698,7 @@ describe('Prebuilt rule schema', () => { ], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -708,8 +708,8 @@ describe('Prebuilt rule schema', () => { }); test('You can send in an array of threat that are missing "technique"', () => { - const payload: Omit & { - threat: Array>>; + const payload: Omit & { + threat: Array>>; } = { ...getPrebuiltRuleMock(), threat: [ @@ -724,33 +724,33 @@ describe('Prebuilt rule schema', () => { ], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You can optionally send in an array of false positives', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), false_positives: ['false_1', 'false_2'], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You cannot send in an array of false positives that are numbers', () => { - const payload: Omit & { + const payload: Omit & { false_positives: number[]; } = { ...getPrebuiltRuleMock(), false_positives: [5, 4], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -760,12 +760,12 @@ describe('Prebuilt rule schema', () => { expect(message.schema).toEqual({}); }); test('You cannot set the risk_score to 101', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), risk_score: 101, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -775,12 +775,12 @@ describe('Prebuilt rule schema', () => { }); test('You cannot set the risk_score to -1', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), risk_score: -1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual(['Invalid value "-1" supplied to "risk_score"']); @@ -788,50 +788,50 @@ describe('Prebuilt rule schema', () => { }); test('You can set the risk_score to 0', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), risk_score: 0, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You can set the risk_score to 100', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), risk_score: 100, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You can set meta to any object you want', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), meta: { somethingMadeUp: { somethingElse: true }, }, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You cannot create meta as a string', () => { - const payload: Omit & { meta: string } = { + const payload: Omit & { meta: string } = { ...getPrebuiltRuleMock(), meta: 'should not work', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -841,25 +841,25 @@ describe('Prebuilt rule schema', () => { }); test('validates with timeline_id and timeline_title', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), timeline_id: 'timeline-id', timeline_title: 'timeline-title', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You cannot set the severity to a value other than low, medium, high, or critical', () => { - const payload: Omit & { severity: string } = { + const payload: Omit & { severity: string } = { ...getPrebuiltRuleMock(), severity: 'junk', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual(['Invalid value "junk" supplied to "severity"']); @@ -867,12 +867,12 @@ describe('Prebuilt rule schema', () => { }); test('You cannot send in an array of actions that are missing "group"', () => { - const payload: Omit = { + const payload: Omit = { ...getPrebuiltRuleMock(), actions: [{ id: 'id', action_type_id: 'action_type_id', params: {} }], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -882,12 +882,12 @@ describe('Prebuilt rule schema', () => { }); test('You cannot send in an array of actions that are missing "id"', () => { - const payload: Omit = { + const payload: Omit = { ...getPrebuiltRuleMock(), actions: [{ group: 'group', action_type_id: 'action_type_id', params: {} }], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -897,12 +897,12 @@ describe('Prebuilt rule schema', () => { }); test('You cannot send in an array of actions that are missing "action_type_id"', () => { - const payload: Omit = { + const payload: Omit = { ...getPrebuiltRuleMock(), actions: [{ group: 'group', id: 'id', params: {} }], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -912,12 +912,12 @@ describe('Prebuilt rule schema', () => { }); test('You cannot send in an array of actions that are missing "params"', () => { - const payload: Omit = { + const payload: Omit = { ...getPrebuiltRuleMock(), actions: [{ group: 'group', id: 'id', action_type_id: 'action_type_id' }], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -927,7 +927,7 @@ describe('Prebuilt rule schema', () => { }); test('You cannot send in an array of actions that are including "actionTypeId"', () => { - const payload: Omit = { + const payload: Omit = { ...getPrebuiltRuleMock(), actions: [ { @@ -939,7 +939,7 @@ describe('Prebuilt rule schema', () => { ], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -950,38 +950,38 @@ describe('Prebuilt rule schema', () => { describe('note', () => { test('You can set note to a string', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), note: '# documentation markdown here', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You can set note to an empty string', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), note: '', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('You cannot create note as an object', () => { - const payload: Omit & { note: {} } = { + const payload: Omit & { note: {} } = { ...getPrebuiltRuleMock(), note: { somethingHere: 'something else', }, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -991,7 +991,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note] does validate', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1006,7 +1006,7 @@ describe('Prebuilt rule schema', () => { version: 1, }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); @@ -1015,7 +1015,7 @@ describe('Prebuilt rule schema', () => { describe('exception_list', () => { test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and exceptions_list] does validate', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1032,14 +1032,14 @@ describe('Prebuilt rule schema', () => { exceptions_list: getListArrayMock(), }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); }); test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, version, and empty exceptions_list] does validate', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1056,7 +1056,7 @@ describe('Prebuilt rule schema', () => { exceptions_list: [], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); @@ -1080,7 +1080,7 @@ describe('Prebuilt rule schema', () => { exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }], }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ @@ -1092,7 +1092,7 @@ describe('Prebuilt rule schema', () => { }); test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and non-existent exceptions_list] does validate with empty exceptions_list', () => { - const payload: PrebuiltRuleToInstall = { + const payload: PrebuiltRuleAsset = { rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1108,7 +1108,7 @@ describe('Prebuilt rule schema', () => { note: '# some markdown', }; - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); @@ -1118,7 +1118,7 @@ describe('Prebuilt rule schema', () => { describe('threat_mapping', () => { test('You can set a threat query, index, mapping, filters on a pre-packaged rule', () => { const payload = getPrebuiltThreatMatchRuleMock(); - const decoded = PrebuiltRuleToInstall.decode(payload); + const decoded = PrebuiltRuleAsset.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts new file mode 100644 index 0000000000000..d1e6029678073 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { + RelatedIntegrationArray, + RequiredFieldArray, + SetupGuide, + RuleSignatureId, + RuleVersion, + BaseCreateProps, + TypeSpecificCreateProps, +} from '../../../../../../common/detection_engine/rule_schema'; + +/** + * Asset containing source content of a prebuilt Security detection rule. + * Is defined for each prebuilt rule in https://github.com/elastic/detection-rules. + * Is shipped via the `security_detection_engine` Fleet package. + * Is installed as saved objects of type "security-rule" when the package is installed. + * + * Additionally, "security-rule" assets can be shipped via other Fleet packages, such as: + * - LotL Attack Detection https://github.com/elastic/integrations/pull/2115 + * - Data Exfiltration Detection + * + * Big differences between this schema and RuleCreateProps: + * - rule_id is required here + * - version is a required field that must exist + */ +export type PrebuiltRuleAsset = t.TypeOf; +export const PrebuiltRuleAsset = t.intersection([ + BaseCreateProps, + TypeSpecificCreateProps, + // version is required here, which supercedes the defaultable version in baseSchema + t.exact( + t.type({ + rule_id: RuleSignatureId, + version: RuleVersion, + }) + ), + t.exact( + t.partial({ + related_integrations: RelatedIntegrationArray, + required_fields: RequiredFieldArray, + setup: SetupGuide, + }) + ), +]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.test.ts similarity index 87% rename from x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.test.ts index 33021d0fc9a1e..1964488442ce7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.test.ts @@ -5,13 +5,13 @@ * 2.0. */ -import type { PrebuiltRuleToInstall } from './prebuilt_rule'; -import { addPrepackagedRuleValidateTypeDependents } from './prebuilt_rule_validate_type_dependents'; -import { getPrebuiltRuleMock } from './prebuilt_rule.mock'; +import type { PrebuiltRuleAsset } from './prebuilt_rule_asset'; +import { addPrepackagedRuleValidateTypeDependents } from './prebuilt_rule_asset_validate_type_dependents'; +import { getPrebuiltRuleMock } from './prebuilt_rule_asset.mock'; describe('addPrepackagedRuleValidateTypeDependents', () => { test('You cannot omit timeline_title when timeline_id is present', () => { - const schema: PrebuiltRuleToInstall = { + const schema: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), timeline_id: '123', }; @@ -21,7 +21,7 @@ describe('addPrepackagedRuleValidateTypeDependents', () => { }); test('You cannot have empty string for timeline_title when timeline_id is present', () => { - const schema: PrebuiltRuleToInstall = { + const schema: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), timeline_id: '123', timeline_title: '', @@ -31,7 +31,7 @@ describe('addPrepackagedRuleValidateTypeDependents', () => { }); test('You cannot have timeline_title with an empty timeline_id', () => { - const schema: PrebuiltRuleToInstall = { + const schema: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), timeline_id: '', timeline_title: 'some-title', @@ -41,7 +41,7 @@ describe('addPrepackagedRuleValidateTypeDependents', () => { }); test('You cannot have timeline_title without timeline_id', () => { - const schema: PrebuiltRuleToInstall = { + const schema: PrebuiltRuleAsset = { ...getPrebuiltRuleMock(), timeline_title: 'some-title', }; @@ -59,7 +59,7 @@ describe('addPrepackagedRuleValidateTypeDependents', () => { value: -1, }, }; - const errors = addPrepackagedRuleValidateTypeDependents(schema as PrebuiltRuleToInstall); + const errors = addPrepackagedRuleValidateTypeDependents(schema as PrebuiltRuleAsset); expect(errors).toEqual(['"threshold.value" has to be bigger than 0']); }); @@ -72,7 +72,7 @@ describe('addPrepackagedRuleValidateTypeDependents', () => { value: 1, }, }; - const errors = addPrepackagedRuleValidateTypeDependents(schema as PrebuiltRuleToInstall); + const errors = addPrepackagedRuleValidateTypeDependents(schema as PrebuiltRuleAsset); expect(errors).toEqual(['Number of fields must be 3 or less']); }); @@ -91,7 +91,7 @@ describe('addPrepackagedRuleValidateTypeDependents', () => { ], }, }; - const errors = addPrepackagedRuleValidateTypeDependents(schema as PrebuiltRuleToInstall); + const errors = addPrepackagedRuleValidateTypeDependents(schema as PrebuiltRuleAsset); expect(errors).toEqual(['Cardinality of a field that is being aggregated on is always 1']); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.ts similarity index 85% rename from x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.ts index 3d9264b3f0556..3ebcad04dbe55 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/prebuilt_rule_validate_type_dependents.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset_validate_type_dependents.ts @@ -5,13 +5,13 @@ * 2.0. */ -import type { PrebuiltRuleToInstall } from './prebuilt_rule'; +import type { PrebuiltRuleAsset } from './prebuilt_rule_asset'; -export const addPrepackagedRuleValidateTypeDependents = (rule: PrebuiltRuleToInstall): string[] => { +export const addPrepackagedRuleValidateTypeDependents = (rule: PrebuiltRuleAsset): string[] => { return [...validateTimelineId(rule), ...validateTimelineTitle(rule), ...validateThreshold(rule)]; }; -const validateTimelineId = (rule: PrebuiltRuleToInstall): string[] => { +const validateTimelineId = (rule: PrebuiltRuleAsset): string[] => { if (rule.timeline_id != null) { if (rule.timeline_title == null) { return ['when "timeline_id" exists, "timeline_title" must also exist']; @@ -24,7 +24,7 @@ const validateTimelineId = (rule: PrebuiltRuleToInstall): string[] => { return []; }; -const validateTimelineTitle = (rule: PrebuiltRuleToInstall): string[] => { +const validateTimelineTitle = (rule: PrebuiltRuleAsset): string[] => { if (rule.timeline_title != null) { if (rule.timeline_id == null) { return ['when "timeline_title" exists, "timeline_id" must also exist']; @@ -37,7 +37,7 @@ const validateTimelineTitle = (rule: PrebuiltRuleToInstall): string[] => { return []; }; -const validateThreshold = (rule: PrebuiltRuleToInstall): string[] => { +const validateThreshold = (rule: PrebuiltRuleAsset): string[] => { const errors: string[] = []; if (rule.type === 'threshold') { if (!rule.threshold) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/get_version_buckets.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/get_version_buckets.ts new file mode 100644 index 0000000000000..9dab6da397b0d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/get_version_buckets.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PrebuiltRuleVersionInfo } from './prebuilt_rule_version_info'; + +export interface GetVersionBucketsArgs { + latestVersions: PrebuiltRuleVersionInfo[]; + installedVersions: PrebuiltRuleVersionInfo[]; +} + +export interface VersionBuckets { + latestVersions: PrebuiltRuleVersionInfo[]; + installedVersions: PrebuiltRuleVersionInfo[]; + latestVersionsToInstall: PrebuiltRuleVersionInfo[]; + latestVersionsToUpgrade: PrebuiltRuleVersionInfo[]; + installedVersionsToUpgrade: PrebuiltRuleVersionInfo[]; +} + +export const getVersionBuckets = (args: GetVersionBucketsArgs): VersionBuckets => { + const { latestVersions, installedVersions } = args; + + const installedVersionsMap = new Map(installedVersions.map((item) => [item.rule_id, item])); + + const latestVersionsToInstall: PrebuiltRuleVersionInfo[] = []; + const latestVersionsToUpgrade: PrebuiltRuleVersionInfo[] = []; + const installedVersionsToUpgrade: PrebuiltRuleVersionInfo[] = []; + + latestVersions.forEach((latestVersion) => { + const installedVersion = installedVersionsMap.get(latestVersion.rule_id); + + if (installedVersion == null) { + // If this rule is not installed + latestVersionsToInstall.push(latestVersion); + } + + if (installedVersion != null && installedVersion.version < latestVersion.version) { + // If this rule is installed but outdated + latestVersionsToUpgrade.push(latestVersion); + installedVersionsToUpgrade.push(installedVersion); + } + }); + + return { + latestVersions, + installedVersions, + latestVersionsToInstall, + latestVersionsToUpgrade, + installedVersionsToUpgrade, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/prebuilt_rule_version_info.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/prebuilt_rule_version_info.ts new file mode 100644 index 0000000000000..bd130917be980 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_versions/prebuilt_rule_version_info.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleSignatureId, + RuleVersion, +} from '../../../../../../common/detection_engine/rule_schema'; + +export interface PrebuiltRuleVersionInfo { + rule_id: RuleSignatureId; + version: RuleVersion; +} diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 7758e74be4801..af26823416a2b 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -90,7 +90,7 @@ export const initRoutes = ( ) => { registerFleetIntegrationsRoutes(router, logger); registerLegacyRuleActionsRoutes(router, logger); - registerPrebuiltRulesRoutes(router, security); + registerPrebuiltRulesRoutes(router, config, security); registerRuleExceptionsRoutes(router); registerManageExceptionsRoutes(router); registerRuleManagementRoutes(router, config, ml, logger); diff --git a/x-pack/plugins/security_solution/server/saved_objects.ts b/x-pack/plugins/security_solution/server/saved_objects.ts index 7ce17ff0c0b51..394cab7f52455 100644 --- a/x-pack/plugins/security_solution/server/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/saved_objects.ts @@ -10,7 +10,7 @@ import type { CoreSetup } from '@kbn/core/server'; import { noteType, pinnedEventType, timelineType } from './lib/timeline/saved_object_mappings'; // eslint-disable-next-line no-restricted-imports import { legacyType as legacyRuleActionsType } from './lib/detection_engine/rule_actions_legacy'; -import { ruleAssetType } from './lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_object_mappings'; +import { prebuiltRuleAssetType } from './lib/detection_engine/prebuilt_rules'; import { type as signalsMigrationType } from './lib/detection_engine/migrations/saved_objects'; import { exceptionsArtifactType, @@ -21,7 +21,7 @@ const types = [ noteType, pinnedEventType, legacyRuleActionsType, - ruleAssetType, + prebuiltRuleAssetType, timelineType, exceptionsArtifactType, manifestType, diff --git a/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/create_prebuilt_rule_saved_objects.ts b/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/create_prebuilt_rule_saved_objects.ts index 974cc4a001247..22cb88687e33b 100644 --- a/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/create_prebuilt_rule_saved_objects.ts +++ b/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/create_prebuilt_rule_saved_objects.ts @@ -6,11 +6,11 @@ */ import { Client } from '@elastic/elasticsearch'; +import { PrebuiltRuleAsset } from '@kbn/security-solution-plugin/server/lib/detection_engine/prebuilt_rules'; import { getPrebuiltRuleMock, getPrebuiltRuleWithExceptionsMock, -} from '@kbn/security-solution-plugin/common/detection_engine/prebuilt_rules/model/prebuilt_rule.mock'; -import { PrebuiltRuleToInstall } from '@kbn/security-solution-plugin/common/detection_engine/prebuilt_rules'; +} from '@kbn/security-solution-plugin/server/lib/detection_engine/prebuilt_rules/mocks'; /** * Rule signature id (`rule.rule_id`) of the prebuilt "Endpoint Security" rule. @@ -23,7 +23,7 @@ export const ELASTIC_SECURITY_RULE_ID = '9a1a2dae-0b5f-4c3d-8305-a268d404c306'; * @param overrideParams Params to override the default mock * @returns Created rule asset saved object */ -export const createRuleAssetSavedObject = (overrideParams: Partial) => ({ +export const createRuleAssetSavedObject = (overrideParams: Partial) => ({ 'security-rule': { ...getPrebuiltRuleMock(), ...overrideParams,