[RAC][Security Solution] Adds migration to new SecuritySolution rule …

…types (#112113) * Initial commit * Properly handle signal history * Fix #95258 - cardinality sort bug * Init threshold rule * Create working threshold rule * Fix threshold signal generation * Fix tests * Update mappings * ALERT_TYPE_ID => RULE_TYPE_ID * Add tests * Fix types * Adds RAC rule type migration * Fix threshold tests (remove outputIndex) * Add threshold rule type to ruleTypeMappings * Add kbn-securitysolution-rules package for sharing with alerting framework * Fix type errors * Fix find_rules tests * First round of test fixes * Fix issues from merge conflicts * Use ruleDataClient getReader() for reading * Fixes to 'generating_signals' tests * Remove more refs to legacy schema * Linting * Quick type fix * Bug fixes * Add saved query rule type * Linting * Fix types * Signal generation tests * Test updates * Update some more refs * build_alert tests * Cleanup * Ref updates * Revert "Ref updates" This reverts commit 4d1473d. * Update status field * Test fixes * Another test * Got a little too aggressive with search/replace * let's see where we're at * Fix * Test fixes * cleanup * Fix cases API integration test config, flaky DE tests * Move flattenWithPrefix to package / skip signal migration tests * Fix unit tests * Use new schema for bulk rule creation * event: { kind } => event.kind * Fix signal migration API tests * Fix ml integration test * Fix threat match integration tests * Fix ML rule type tests and add correct producer to all rule types * Update threat match API integration test * Remove dupe properties * Type fix * Fix ML producer in functional test * Fix generating_signals tests * Remove usage of RuleDataClient-based execution log client * Don't check output index version if rule registry enabled * Fix bulk duplicate rule * Fix duplicate rule test * Fix readPrivileges and timestamp check logic * Fixes for eql and exceptions tests... disable open_close_signals * Type fixes / keyword test fixes * Additional test fixes * Unit test fixes + signal -> kibana.alert * Test fixes for exceptions * Fix read_resolve_rules test * Various test fixes with marshallmain * Sort search results * Fix create_rules tests * Disable writer cache for integration tests * Disable writer cache for cases integration tests * Fix types in rule_data_plugin_service * Fix ordering in exceptions tests * Remove rule_registry.enabled flag * Fix signals migration tests * Don't check signals index before creation * Fix cypress config * Fix type error * create_migrations tests * Skip flaky test * Helpful comment * Fixes from merge conflicts * Pretend that signals index exists * Fix type errors * Skip flaky tests * Fix threat matching test * Clean up * Reverting default ruleRegistry experimental flag (breaks unit tests) * Reenable rule registry experimental feature by default * Execute DE rule migration in 8.0 Co-authored-by: Marshall Main <[email protected]>
elastic · Oct 26, 2021 · 117efdf · 117efdf
1 parent f92cbbc
commit 117efdf
Show file tree

Hide file tree

Showing 188 changed files with 2,620 additions and 4,596 deletions.
diff --git a/docs/developer/getting-started/monorepo-packages.asciidoc b/docs/developer/getting-started/monorepo-packages.asciidoc
@@ -91,6 +91,7 @@ yarn kbn watch
 - @kbn/securitysolution-list-constants
 - @kbn/securitysolution-list-hooks
 - @kbn/securitysolution-list-utils
+- @kbn/securitysolution-rules
 - @kbn/securitysolution-utils
 - @kbn/server-http-tools
 - @kbn/server-route-repository

diff --git a/package.json b/package.json
@@ -148,6 +148,7 @@
     "@kbn/securitysolution-list-constants": "link:bazel-bin/packages/kbn-securitysolution-list-constants",
     "@kbn/securitysolution-list-hooks": "link:bazel-bin/packages/kbn-securitysolution-list-hooks",
     "@kbn/securitysolution-list-utils": "link:bazel-bin/packages/kbn-securitysolution-list-utils",
+    "@kbn/securitysolution-rules": "link:bazel-bin/packages/kbn-securitysolution-rules",
     "@kbn/securitysolution-t-grid": "link:bazel-bin/packages/kbn-securitysolution-t-grid",
     "@kbn/securitysolution-utils": "link:bazel-bin/packages/kbn-securitysolution-utils",
     "@kbn/server-http-tools": "link:bazel-bin/packages/kbn-server-http-tools",

diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel
@@ -46,6 +46,7 @@ filegroup(
       "//packages/kbn-securitysolution-list-api:build",
       "//packages/kbn-securitysolution-list-hooks:build",
       "//packages/kbn-securitysolution-list-utils:build",
+      "//packages/kbn-securitysolution-rules:build",
       "//packages/kbn-securitysolution-utils:build",
       "//packages/kbn-securitysolution-es-utils:build",
       "//packages/kbn-securitysolution-t-grid:build",

diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts
@@ -17,6 +17,7 @@ const CONSUMERS = `${KIBANA_NAMESPACE}.consumers` as const;
 const ECS_VERSION = 'ecs.version' as const;
 const EVENT_ACTION = 'event.action' as const;
 const EVENT_KIND = 'event.kind' as const;
+const EVENT_MODULE = 'event.module' as const;
 const SPACE_IDS = `${KIBANA_NAMESPACE}.space_ids` as const;
 const TAGS = 'tags' as const;
 const TIMESTAMP = '@timestamp' as const;
@@ -88,6 +89,7 @@ const fields = {
   ECS_VERSION,
   EVENT_KIND,
   EVENT_ACTION,
+  EVENT_MODULE,
   TAGS,
   TIMESTAMP,
   ALERT_ACTION_GROUP,
@@ -189,6 +191,7 @@ export {
   ECS_VERSION,
   EVENT_ACTION,
   EVENT_KIND,
+  EVENT_MODULE,
   KIBANA_NAMESPACE,
   ALERT_RULE_UUID,
   ALERT_RULE_CATEGORY,

diff --git a/packages/kbn-securitysolution-rules/BUILD.bazel b/packages/kbn-securitysolution-rules/BUILD.bazel
@@ -0,0 +1,95 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler")
+
+PKG_BASE_NAME = "kbn-securitysolution-rules"
+
+PKG_REQUIRE_NAME = "@kbn/securitysolution-rules"
+
+SOURCE_FILES = glob(
+  [
+    "src/**/*.ts",
+  ],
+  exclude = [
+    "**/*.test.*",
+    "**/*.mock.*",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+  "README.md",
+]
+
+RUNTIME_DEPS = [
+  "@npm//lodash",
+  "@npm//tslib",
+  "@npm//uuid",
+]
+
+TYPES_DEPS = [
+  "@npm//tslib",
+  "@npm//@types/jest",
+  "@npm//@types/lodash",
+  "@npm//@types/node",
+  "@npm//@types/uuid"
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ["--pretty"],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  declaration_map = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  root_dir = "src",
+  source_map = True,
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_BASE_NAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node", ":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [
+      ":%s" % PKG_BASE_NAME,
+  ],
+)
+
+filegroup(
+  name = "build",
+  srcs = [
+      ":npm_module",
+  ],
+  visibility = ["//visibility:public"],
+)
diff --git a/packages/kbn-securitysolution-rules/README.md b/packages/kbn-securitysolution-rules/README.md
@@ -0,0 +1,3 @@
+# kbn-securitysolution-rules
+
+This contains alerts-as-data rule-specific constants and mappings that can be used across plugins.
diff --git a/packages/kbn-securitysolution-rules/jest.config.js b/packages/kbn-securitysolution-rules/jest.config.js
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-securitysolution-rules'],
+};
diff --git a/packages/kbn-securitysolution-rules/package.json b/packages/kbn-securitysolution-rules/package.json
@@ -0,0 +1,9 @@
+{
+  "name": "@kbn/securitysolution-rules",
+  "version": "1.0.0",
+  "description": "security solution rule utilities to use across plugins",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "main": "./target_node/index.js",
+  "types": "./target_types/index.d.ts",
+  "private": true
+}
diff --git a/packages/kbn-securitysolution-rules/src/index.ts b/packages/kbn-securitysolution-rules/src/index.ts
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './rule_type_constants';
+export * from './rule_type_mappings';
+export * from './utils';
diff --git a/packages/kbn-securitysolution-rules/src/rule_type_constants.ts b/packages/kbn-securitysolution-rules/src/rule_type_constants.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * Id for the legacy siem signals alerting type
+ */
+export const SIGNALS_ID = `siem.signals` as const;
+
+/**
+ * IDs for alerts-as-data rule types
+ */
+const RULE_TYPE_PREFIX = `siem` as const;
+export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const;
+export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const;
+export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const;
+export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const;
+export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const;
+export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const;
diff --git a/packages/kbn-securitysolution-rules/src/rule_type_mappings.ts b/packages/kbn-securitysolution-rules/src/rule_type_mappings.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+  EQL_RULE_TYPE_ID,
+  INDICATOR_RULE_TYPE_ID,
+  ML_RULE_TYPE_ID,
+  QUERY_RULE_TYPE_ID,
+  SAVED_QUERY_RULE_TYPE_ID,
+  THRESHOLD_RULE_TYPE_ID,
+} from './rule_type_constants';
+
+/**
+ * Maps legacy rule types to RAC rule type IDs.
+ */
+export const ruleTypeMappings = {
+  eql: EQL_RULE_TYPE_ID,
+  machine_learning: ML_RULE_TYPE_ID,
+  query: QUERY_RULE_TYPE_ID,
+  saved_query: SAVED_QUERY_RULE_TYPE_ID,
+  threat_match: INDICATOR_RULE_TYPE_ID,
+  threshold: THRESHOLD_RULE_TYPE_ID,
+};
+type RuleTypeMappings = typeof ruleTypeMappings;
+
+export type RuleType = keyof RuleTypeMappings;
+export type RuleTypeId = RuleTypeMappings[keyof RuleTypeMappings];
diff --git a/...es/factories/utils/flatten_with_prefix.ts → ...s/kbn-securitysolution-rules/src/utils.ts b/...es/factories/utils/flatten_with_prefix.ts → ...s/kbn-securitysolution-rules/src/utils.ts
@@ -1,12 +1,23 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
 import { isPlainObject } from 'lodash';
-import { SearchTypes } from '../../../../../../common/detection_engine/types';
+import { RuleType, RuleTypeId, ruleTypeMappings } from './rule_type_mappings';
+
+export const isRuleType = (ruleType: unknown): ruleType is RuleType => {
+  return Object.keys(ruleTypeMappings).includes(ruleType as string);
+};
+
+export const isRuleTypeId = (ruleTypeId: unknown): ruleTypeId is RuleTypeId => {
+  return Object.values(ruleTypeMappings).includes(ruleTypeId as RuleTypeId);
+};
+
+type SearchTypes = string | number | boolean | object | SearchTypes[] | undefined;
 
 export const flattenWithPrefix = (
   prefix: string,

diff --git a/packages/kbn-securitysolution-rules/tsconfig.json b/packages/kbn-securitysolution-rules/tsconfig.json
@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "rootDir": "src",
+    "sourceMap": true,
+    "sourceRoot": "../../../../packages/kbn-securitysolution-rules/src",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ]
+}
diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { isRuleType, ruleTypeMappings } from '@kbn/securitysolution-rules';
 import { isString } from 'lodash/fp';
 import {
   LogMeta,
@@ -52,7 +53,8 @@ export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc<RawAl
     SUPPORT_INCIDENTS_ACTION_TYPES.includes(action.actionTypeId)
   );
 
-export const isSecuritySolutionRule = (doc: SavedObjectUnsanitizedDoc<RawAlert>): boolean =>
+// Deprecated in 8.0
+export const isSiemSignalsRuleType = (doc: SavedObjectUnsanitizedDoc<RawAlert>): boolean =>
   doc.attributes.alertTypeId === 'siem.signals';
 
 /**
@@ -96,19 +98,19 @@ export function getMigrations(
 
   const migrationSecurityRules713 = createEsoMigration(
     encryptedSavedObjects,
-    (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
+    (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSiemSignalsRuleType(doc),
     pipeMigrations(removeNullsFromSecurityRules)
   );
 
   const migrationSecurityRules714 = createEsoMigration(
     encryptedSavedObjects,
-    (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
+    (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSiemSignalsRuleType(doc),
     pipeMigrations(removeNullAuthorFromSecurityRules)
   );
 
   const migrationSecurityRules715 = createEsoMigration(
     encryptedSavedObjects,
-    (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
+    (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSiemSignalsRuleType(doc),
     pipeMigrations(addExceptionListsToReferences)
   );
 
@@ -126,7 +128,7 @@ export function getMigrations(
   const migrationRules800 = createEsoMigration(
     encryptedSavedObjects,
     (doc: SavedObjectUnsanitizedDoc<RawAlert>): doc is SavedObjectUnsanitizedDoc<RawAlert> => true,
-    (doc) => doc // no-op
+    pipeMigrations(addRACRuleTypes)
   );
 
   return {
@@ -647,6 +649,25 @@ function setLegacyId(
   };
 }
 
+function addRACRuleTypes(
+  doc: SavedObjectUnsanitizedDoc<RawAlert>
+): SavedObjectUnsanitizedDoc<RawAlert> {
+  const ruleType = doc.attributes.params.type;
+  return isSiemSignalsRuleType(doc) && isRuleType(ruleType)
+    ? {
+        ...doc,
+        attributes: {
+          ...doc.attributes,
+          alertTypeId: ruleTypeMappings[ruleType],
+          params: {
+            ...doc.attributes.params,
+            outputIndex: '',
+          },
+        },
+      }
+    : doc;
+}
+
 function getRemovePreconfiguredConnectorsFromReferencesFn(
   isPreconfigured: (connectorId: string) => boolean
 ) {
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# kbn-securitysolution-rules

		This contains alerts-as-data rule-specific constants and mappings that can be used across plugins.