[SecuritySolution][SIEM Migrations] Rule migrations storage

.github/CODEOWNERS

@@ -1609,7 +1609,12 @@ x-pack/test/security_solution_api_integration/test_suites/sources @elastic/secur
 /x-pack/test/security_solution_playwright @elastic/security-engineering-productivity
 /x-pack/plugins/security_solution/scripts/run_cypress @MadameSheema @patrykkopycinski @maximpn @banderror
 
-## Security Solution sub teams - Threat Hunting Investigations
+## Security Solution sub teams - Threat Hunting
+
+/x-pack/plugins/security_solution/server/lib/siem_migrations @elastic/security-threat-hunting
+/x-pack/plugins/security_solution/common/siem_migrations @elastic/security-threat-hunting
+
+## Security Solution Threat Hunting areas - Threat Hunting Investigations
 
 /x-pack/plugins/security_solution/common/api/timeline @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/common/search_strategy/timeline @elastic/security-threat-hunting-investigations
@@ -1639,7 +1644,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 
 /x-pack/plugins/security_solution/server/lib/timeline @elastic/security-threat-hunting-investigations
 
-## Security Solution sub teams - Threat Hunting Explore
+## Security Solution Threat Hunting areas - Threat Hunting Explore
 /x-pack/plugins/security_solution/common/api/tags @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram @elastic/security-threat-hunting-explore

packages/kbn-data-stream-adapter/src/field_maps/types.ts

@@ -38,8 +38,9 @@ export interface EcsMetadata {
   properties?: Record<string, { type: string }>;
 }
 
-export interface FieldMap {
-  [key: string]: {
+export type FieldMap<T extends string = string> = Record<
+  T,
+  {
     type: string;
     required: boolean;
     array?: boolean;
@@ -53,5 +54,17 @@ export interface FieldMap {
     scaling_factor?: number;
     dynamic?: boolean | 'strict';
     properties?: Record<string, { type: string }>;
-  };
-}
+  }
+>;
+
+// This utility type flattens all the keys of a schema object and its nested objects as a union type.
+// Its purpose is to ensure that the FieldMap keys are always in sync with the schema object.
+// It assumes all optional fields of the schema are required in the field map, they can always be omitted from the resulting type.
+export type SchemaFieldMapKeys<
+  T extends Record<string, unknown>,
+  Key = keyof T
+> = Key extends string
+  ? NonNullable<T[Key]> extends Record<string, unknown>
+    ? `${Key}` | `${Key}.${SchemaFieldMapKeys<NonNullable<T[Key]>>}`
+    : `${Key}`
+  : never;

x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts

@@ -361,6 +361,11 @@ import type {
   ResolveTimelineRequestQueryInput,
   ResolveTimelineResponse,
 } from './timeline/resolve_timeline/resolve_timeline_route.gen';
+import type {
+  CreateRuleMigrationRequestBodyInput,
+  CreateRuleMigrationResponse,
+  GetRuleMigrationResponse,
+} from '../siem_migrations/model/api/rules/rules_migration.gen';
 
 export interface ClientOptions {
   kbnClient: KbnClient;
@@ -655,6 +660,22 @@ If a record already exists for the specified entity, that record is overwritten
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Creates a new SIEM rules migration using the original vendor rules provided
+   */
+  async createRuleMigration(props: CreateRuleMigrationProps) {
+    this.log.info(`${new Date().toISOString()} Calling API CreateRuleMigration`);
+    return this.kbnClient
+      .request<CreateRuleMigrationResponse>({
+        path: '/internal/siem_migrations/rules',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'POST',
+        body: props.body,
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   /**
    * Create a new Timeline or Timeline template.
    */
@@ -1396,6 +1417,21 @@ finalize it.
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Retrieves the rule migrations stored in the system
+   */
+  async getRuleMigration() {
+    this.log.info(`${new Date().toISOString()} Calling API GetRuleMigration`);
+    return this.kbnClient
+      .request<GetRuleMigrationResponse>({
+        path: '/internal/siem_migrations/rules',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'GET',
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   /**
    * Get the details of an existing saved Timeline or Timeline template.
    */
@@ -2039,6 +2075,9 @@ export interface CreateAssetCriticalityRecordProps {
 export interface CreateRuleProps {
   body: CreateRuleRequestBodyInput;
 }
+export interface CreateRuleMigrationProps {
+  body: CreateRuleMigrationRequestBodyInput;
+}
 export interface CreateTimelinesProps {
   body: CreateTimelinesRequestBodyInput;
 }

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -235,6 +235,11 @@ export const allowedExperimentalValues = Object.freeze({
    * can be disabled if necessary in a given environment.
    */
   entityStoreDisabled: false,
+
+  /**
+   * Enables the siem migrations feature
+   */
+  siemMigrationsEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

x-pack/plugins/security_solution/common/siem_migrations/constants.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const;
+export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const;
+
+export enum SiemMigrationsStatus {
+  PENDING = 'pending',
+  PROCESSING = 'processing',
+  FINISHED = 'finished',
+  ERROR = 'error',
+}

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Common SIEM Migrations Attributes
+ *   version: not applicable
+ */
+
+import { z } from '@kbn/zod';
+
+/**
+ * The GenAI connector id to use.
+ */
+export type ConnectorId = z.infer<typeof ConnectorId>;
+export const ConnectorId = z.string();
+
+/**
+ * The LangSmith options object.
+ */
+export type LangSmithOptions = z.infer<typeof LangSmithOptions>;
+export const LangSmithOptions = z.object({
+  /**
+   * The project name.
+   */
+  project_name: z.string(),
+  /**
+   * The API key to use for tracing.
+   */
+  api_key: z.string(),
+});

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml

@@ -0,0 +1,24 @@
+openapi: 3.0.3
+info:
+  title: Common SIEM Migrations Attributes
+  version: 'not applicable'
+paths: {}
+components:
+  x-codegen-enabled: true
+  schemas:
+    ConnectorId:
+      type: string
+      description: The GenAI connector id to use.
+    LangSmithOptions:
+      type: object
+      description: The LangSmith options object.
+      required:
+        - project_name
+        - api_key
+      properties:
+        project_name:
+          type: string
+          description: The project name.
+        api_key:
+          type: string
+          description: The API key to use for tracing.

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: SIEM Rules Migration API endpoint
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
+
+import { OriginalRule, RuleMigration } from '../../rule_migration.gen';
+
+export type CreateRuleMigrationRequestBody = z.infer<typeof CreateRuleMigrationRequestBody>;
+export const CreateRuleMigrationRequestBody = z.array(OriginalRule);
+export type CreateRuleMigrationRequestBodyInput = z.input<typeof CreateRuleMigrationRequestBody>;
+
+export type CreateRuleMigrationResponse = z.infer<typeof CreateRuleMigrationResponse>;
+export const CreateRuleMigrationResponse = z.object({
+  /**
+   * The migration id created.
+   */
+  migration_id: z.string(),
+});
+
+export type GetRuleMigrationResponse = z.infer<typeof GetRuleMigrationResponse>;
+export const GetRuleMigrationResponse = z.array(RuleMigration);

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml

@@ -0,0 +1,52 @@
+openapi: 3.0.3
+info:
+  title: SIEM Rules Migration API endpoint
+  version: '1'
+paths:
+  /internal/siem_migrations/rules:
+    post:
+      summary: Creates a new rule migration
+      operationId: CreateRuleMigration
+      x-codegen-enabled: true
+      description: Creates a new SIEM rules migration using the original vendor rules provided
+      tags:
+        - SIEM Migrations
+        - Rule Migrations
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: '../../rule_migration.schema.yaml#/components/schemas/OriginalRule'
+      responses:
+        200:
+          description: Indicates migration have been created correctly.
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - migration_id
+                properties:
+                  migration_id:
+                    type: string
+                    description: The migration id created.
+    get:
+      summary: Retrieves rule migrations
+      operationId: GetRuleMigration
+      x-codegen-enabled: true
+      description: Retrieves the rule migrations stored in the system
+      tags:
+        - SIEM Migrations
+        - Rule Migrations
+      responses:
+        200:
+          description: Indicates rule migrations have been retrieved correctly.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration'