diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index 065601f1cf4ab..ca28715a1524e 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -364,7 +364,16 @@ import type { import type { CreateRuleMigrationRequestBodyInput, CreateRuleMigrationResponse, + GetAllStatsRuleMigrationResponse, + GetRuleMigrationRequestParamsInput, GetRuleMigrationResponse, + GetRuleMigrationStatsRequestParamsInput, + GetRuleMigrationStatsResponse, + StartRuleMigrationRequestParamsInput, + StartRuleMigrationRequestBodyInput, + StartRuleMigrationResponse, + StopRuleMigrationRequestParamsInput, + StopRuleMigrationResponse, } from '../siem_migrations/model/api/rules/rules_migration.gen'; export interface ClientOptions { @@ -1205,6 +1214,21 @@ finalize it. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Retrieves the rule migrations stats for all migrations stored in the system + */ + async getAllStatsRuleMigration() { + this.log.info(`${new Date().toISOString()} Calling API GetAllStatsRuleMigration`); + return this.kbnClient + .request({ + path: '/internal/siem_migrations/rules/stats', + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Get the criticality record for a specific asset. */ @@ -1395,13 +1419,28 @@ finalize it. .catch(catchAxiosErrorFormatAndThrow); } /** - * Retrieves the rule migrations stored in the system + * Retrieves the rule documents stored in the system given the rule migration id */ - async getRuleMigration() { + async getRuleMigration(props: GetRuleMigrationProps) { this.log.info(`${new Date().toISOString()} Calling API GetRuleMigration`); return this.kbnClient .request({ - path: '/internal/siem_migrations/rules', + path: replaceParams('/internal/siem_migrations/rules/{migration_id}', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } + /** + * Retrieves the stats of a SIEM rules migration using the migration id provided + */ + async getRuleMigrationStats(props: GetRuleMigrationStatsProps) { + this.log.info(`${new Date().toISOString()} Calling API GetRuleMigrationStats`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/stats', props.params), headers: { [ELASTIC_HTTP_VERSION_HEADER]: '1', }, @@ -1913,6 +1952,22 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Starts a SIEM rules migration using the migration id provided + */ + async startRuleMigration(props: StartRuleMigrationProps) { + this.log.info(`${new Date().toISOString()} Calling API StartRuleMigration`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/start', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'PUT', + body: props.body, + }) + .catch(catchAxiosErrorFormatAndThrow); + } async stopEntityEngine(props: StopEntityEngineProps) { this.log.info(`${new Date().toISOString()} Calling API StopEntityEngine`); return this.kbnClient @@ -1925,6 +1980,21 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Stops a running SIEM rules migration using the migration id provided + */ + async stopRuleMigration(props: StopRuleMigrationProps) { + this.log.info(`${new Date().toISOString()} Calling API StopRuleMigration`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/stop', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'PUT', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Suggests user profiles. */ @@ -2161,6 +2231,12 @@ export interface GetRuleExecutionResultsProps { query: GetRuleExecutionResultsRequestQueryInput; params: GetRuleExecutionResultsRequestParamsInput; } +export interface GetRuleMigrationProps { + params: GetRuleMigrationRequestParamsInput; +} +export interface GetRuleMigrationStatsProps { + params: GetRuleMigrationStatsRequestParamsInput; +} export interface GetTimelineProps { query: GetTimelineRequestQueryInput; } @@ -2237,9 +2313,16 @@ export interface SetAlertTagsProps { export interface StartEntityEngineProps { params: StartEntityEngineRequestParamsInput; } +export interface StartRuleMigrationProps { + params: StartRuleMigrationRequestParamsInput; + body: StartRuleMigrationRequestBodyInput; +} export interface StopEntityEngineProps { params: StopEntityEngineRequestParamsInput; } +export interface StopRuleMigrationProps { + params: StopRuleMigrationRequestParamsInput; +} export interface SuggestUserProfilesProps { query: SuggestUserProfilesRequestQueryInput; } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index 96ca75679f112..f2efc646a8101 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -8,9 +8,24 @@ export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const; export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const; -export enum SiemMigrationsStatus { +export const SIEM_RULE_MIGRATIONS_ALL_STATS_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/stats` as const; +export const SIEM_RULE_MIGRATIONS_GET_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}` as const; +export const SIEM_RULE_MIGRATIONS_START_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/start` as const; +export const SIEM_RULE_MIGRATIONS_STATS_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stats` as const; +export const SIEM_RULE_MIGRATIONS_STOP_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stop` as const; + +export enum SiemMigrationStatus { PENDING = 'pending', PROCESSING = 'processing', - FINISHED = 'finished', - ERROR = 'error', + COMPLETED = 'completed', + FAILED = 'failed', +} + +export enum SiemMigrationRuleTranslationResult { + FULL = 'full', + PARTIAL = 'partial', + UNTRANSLATABLE = 'untranslatable', } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts index fa8a1cc8a6778..120505ec43cb7 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts @@ -16,7 +16,13 @@ import { z } from '@kbn/zod'; -import { OriginalRule, RuleMigration } from '../../rule_migration.gen'; +import { + OriginalRule, + RuleMigrationAllTaskStats, + RuleMigration, + RuleMigrationTaskStats, +} from '../../rule_migration.gen'; +import { ConnectorId, LangSmithOptions } from '../common.gen'; export type CreateRuleMigrationRequestBody = z.infer; export const CreateRuleMigrationRequestBody = z.array(OriginalRule); @@ -30,5 +36,60 @@ export const CreateRuleMigrationResponse = z.object({ migration_id: z.string(), }); +export type GetAllStatsRuleMigrationResponse = z.infer; +export const GetAllStatsRuleMigrationResponse = RuleMigrationAllTaskStats; + +export type GetRuleMigrationRequestParams = z.infer; +export const GetRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type GetRuleMigrationRequestParamsInput = z.input; + export type GetRuleMigrationResponse = z.infer; export const GetRuleMigrationResponse = z.array(RuleMigration); + +export type GetRuleMigrationStatsRequestParams = z.infer; +export const GetRuleMigrationStatsRequestParams = z.object({ + migration_id: z.string(), +}); +export type GetRuleMigrationStatsRequestParamsInput = z.input< + typeof GetRuleMigrationStatsRequestParams +>; + +export type GetRuleMigrationStatsResponse = z.infer; +export const GetRuleMigrationStatsResponse = RuleMigrationTaskStats; + +export type StartRuleMigrationRequestParams = z.infer; +export const StartRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type StartRuleMigrationRequestParamsInput = z.input; + +export type StartRuleMigrationRequestBody = z.infer; +export const StartRuleMigrationRequestBody = z.object({ + connector_id: ConnectorId, + langsmith_options: LangSmithOptions.optional(), +}); +export type StartRuleMigrationRequestBodyInput = z.input; + +export type StartRuleMigrationResponse = z.infer; +export const StartRuleMigrationResponse = z.object({ + /** + * Indicates the migration has been started. `false` means the migration does not need to be started. + */ + started: z.boolean(), +}); + +export type StopRuleMigrationRequestParams = z.infer; +export const StopRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type StopRuleMigrationRequestParamsInput = z.input; + +export type StopRuleMigrationResponse = z.infer; +export const StopRuleMigrationResponse = z.object({ + /** + * Indicates the migration has been stopped. + */ + stopped: z.boolean(), +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml index 40596ba7e712d..7b06c3d6a22ac 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml @@ -10,8 +10,7 @@ paths: x-codegen-enabled: true description: Creates a new SIEM rules migration using the original vendor rules provided tags: - - SIEM Migrations - - Rule Migrations + - SIEM Rule Migrations requestBody: required: true content: @@ -33,20 +32,146 @@ paths: migration_id: type: string description: The migration id created. + + /internal/siem_migrations/rules/stats: get: - summary: Retrieves rule migrations - operationId: GetRuleMigration + summary: Retrieves the stats for all rule migrations + operationId: GetAllStatsRuleMigration x-codegen-enabled: true - description: Retrieves the rule migrations stored in the system + description: Retrieves the rule migrations stats for all migrations stored in the system tags: - - SIEM Migrations - - Rule Migrations + - SIEM Rule Migrations responses: 200: description: Indicates rule migrations have been retrieved correctly. + content: + application/json: + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationAllTaskStats' + + /internal/siem_migrations/rules/{migration_id}: + get: + summary: Retrieves all the rules of a migration + operationId: GetRuleMigration + x-codegen-enabled: true + description: Retrieves the rule documents stored in the system given the rule migration id + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + responses: + 200: + description: Indicates rule migration have been retrieved correctly. content: application/json: schema: type: array items: $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration' + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/start: + put: + summary: Starts a rule migration + operationId: StartRuleMigration + x-codegen-enabled: true + description: Starts a SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - connector_id + properties: + connector_id: + $ref: '../common.schema.yaml#/components/schemas/ConnectorId' + langsmith_options: + $ref: '../common.schema.yaml#/components/schemas/LangSmithOptions' + responses: + 200: + description: Indicates the migration start request has been processed successfully. + content: + application/json: + schema: + type: object + required: + - started + properties: + started: + type: boolean + description: Indicates the migration has been started. `false` means the migration does not need to be started. + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/stats: + get: + summary: Gets a rule migration task stats + operationId: GetRuleMigrationStats + x-codegen-enabled: true + description: Retrieves the stats of a SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + responses: + 200: + description: Indicates the migration stats has been retrieved correctly. + content: + application/json: + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationTaskStats' + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/stop: + put: + summary: Stops an existing rule migration + operationId: StopRuleMigration + x-codegen-enabled: true + description: Stops a running SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to stop + responses: + 200: + description: Indicates migration task stop has been processed successfully. + content: + application/json: + schema: + type: object + required: + - stopped + properties: + stopped: + type: boolean + description: Indicates the migration has been stopped. + 204: + description: Indicates the migration id was not found running. diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 0e07ef2f208da..fe00c4b4df1c6 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -71,11 +71,11 @@ export const ElasticRule = z.object({ /** * The translated elastic query. */ - query: z.string(), + query: z.string().optional(), /** * The translated elastic query language. */ - query_language: z.literal('esql').default('esql'), + query_language: z.literal('esql').optional(), /** * The Elastic prebuilt rule id matched. */ @@ -99,16 +99,20 @@ export const RuleMigration = z.object({ * The migration id. */ migration_id: z.string(), + /** + * The username of the user who created the migration. + */ + created_by: z.string(), original_rule: OriginalRule, elastic_rule: ElasticRule.optional(), /** - * The translation state. + * The rule translation result. */ - translation_state: z.enum(['complete', 'partial', 'untranslatable']).optional(), + translation_result: z.enum(['full', 'partial', 'untranslatable']).optional(), /** - * The status of the rule migration. + * The status of the rule migration process. */ - status: z.enum(['pending', 'processing', 'finished', 'error']).default('pending'), + status: z.enum(['pending', 'processing', 'completed', 'failed']).default('pending'), /** * The comments for the migration including a summary from the LLM in markdown. */ @@ -122,3 +126,55 @@ export const RuleMigration = z.object({ */ updated_by: z.string().optional(), }); + +/** + * The rule migration task stats object. + */ +export type RuleMigrationTaskStats = z.infer; +export const RuleMigrationTaskStats = z.object({ + /** + * Indicates if the migration task status. + */ + status: z.enum(['ready', 'running', 'stopped', 'finished']), + /** + * The rules migration stats. + */ + rules: z.object({ + /** + * The total number of rules to migrate. + */ + total: z.number().int(), + /** + * The number of rules that are pending migration. + */ + pending: z.number().int(), + /** + * The number of rules that are being migrated. + */ + processing: z.number().int(), + /** + * The number of rules that have been migrated successfully. + */ + completed: z.number().int(), + /** + * The number of rules that have failed migration. + */ + failed: z.number().int(), + }), + /** + * The moment of the last update. + */ + last_updated_at: z.string().optional(), +}); + +export type RuleMigrationAllTaskStats = z.infer; +export const RuleMigrationAllTaskStats = z.array( + RuleMigrationTaskStats.merge( + z.object({ + /** + * The migration id + */ + migration_id: z.string(), + }) + ) +); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 9ec825389a52b..c9841856a6914 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -48,8 +48,6 @@ components: description: The migrated elastic rule. required: - title - - query - - query_language properties: title: type: string @@ -68,7 +66,6 @@ components: description: The translated elastic query language. enum: - esql - default: esql prebuilt_rule_id: type: string description: The Elastic prebuilt rule id matched. @@ -84,32 +81,36 @@ components: - migration_id - original_rule - status + - created_by properties: - "@timestamp": + '@timestamp': type: string description: The moment of creation migration_id: type: string description: The migration id. + created_by: + type: string + description: The username of the user who created the migration. original_rule: $ref: '#/components/schemas/OriginalRule' elastic_rule: $ref: '#/components/schemas/ElasticRule' - translation_state: + translation_result: type: string - description: The translation state. - enum: - - complete + description: The rule translation result. + enum: # should match SiemMigrationRuleTranslationResult enum at ../constants.ts + - full - partial - untranslatable status: type: string - description: The status of the rule migration. + description: The status of the rule migration process. enum: # should match SiemMigrationsStatus enum at ../constants.ts - pending - processing - - finished - - error + - completed + - failed default: pending comments: type: array @@ -122,3 +123,60 @@ components: updated_by: type: string description: The user who last updated the migration + + RuleMigrationTaskStats: + type: object + description: The rule migration task stats object. + required: + - status + - rules + properties: + status: + type: string + description: Indicates if the migration task status. + enum: + - ready + - running + - stopped + - finished + rules: + type: object + description: The rules migration stats. + required: + - total + - pending + - processing + - completed + - failed + properties: + total: + type: integer + description: The total number of rules to migrate. + pending: + type: integer + description: The number of rules that are pending migration. + processing: + type: integer + description: The number of rules that are being migrated. + completed: + type: integer + description: The number of rules that have been migrated successfully. + failed: + type: integer + description: The number of rules that have failed migration. + last_updated_at: + type: string + description: The moment of the last update. + + RuleMigrationAllTaskStats: + type: array + items: + allOf: + - $ref: '#/components/schemas/RuleMigrationTaskStats' + - type: object + required: + - migration_id + properties: + migration_id: + type: string + description: The migration id diff --git a/x-pack/plugins/security_solution/kibana.jsonc b/x-pack/plugins/security_solution/kibana.jsonc index e48a9794b7e5c..8c8b77d48bc9f 100644 --- a/x-pack/plugins/security_solution/kibana.jsonc +++ b/x-pack/plugins/security_solution/kibana.jsonc @@ -54,7 +54,8 @@ "savedSearch", "unifiedDocViewer", "charts", - "entityManager" + "entityManager", + "inference" ], "optionalPlugins": [ "encryptedSavedObjects", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts index ebc1706b309f8..d2aacbdeaeeaf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts @@ -79,7 +79,8 @@ export const createMockClients = () => { internalFleetServices: { packages: packageServiceMock.createClient(), }, - siemMigrationsClient: siemMigrationsServiceMock.createClient(), + siemRuleMigrationsClient: siemMigrationsServiceMock.createRulesClient(), + getInferenceClient: jest.fn(), }; }; @@ -165,7 +166,8 @@ const createSecuritySolutionRequestContextMock = ( getAssetCriticalityDataClient: jest.fn(() => clients.assetCriticalityDataClient), getAuditLogger: jest.fn(() => mockAuditLogger), getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient), - getSiemMigrationsClient: jest.fn(() => clients.siemMigrationsClient), + getSiemRuleMigrationsClient: jest.fn(() => clients.siemRuleMigrationsClient), + getInferenceClient: jest.fn(() => clients.getInferenceClient()), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts index fcf119e19ece5..af961d48db5b1 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts @@ -7,18 +7,16 @@ import { createRuleMigrationClient } from '../rules/__mocks__/mocks'; -const createClient = () => ({ rules: createRuleMigrationClient() }); - export const mockSetup = jest.fn().mockResolvedValue(undefined); -export const mockCreateClient = jest.fn().mockReturnValue(createClient()); +export const mockCreateClient = jest.fn().mockReturnValue(createRuleMigrationClient()); export const mockStop = jest.fn(); export const siemMigrationsServiceMock = { create: () => jest.fn().mockImplementation(() => ({ setup: mockSetup, - createClient: mockCreateClient, + createRulesClient: mockCreateClient, stop: mockStop, })), - createClient: () => createClient(), + createRulesClient: () => createRuleMigrationClient(), }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts new file mode 100644 index 0000000000000..659929d47570f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { siemMigrationsServiceMock } from './mocks'; +export const SiemMigrationsService = siemMigrationsServiceMock.create(); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts index 8233151f513e4..8811a54195e2b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts @@ -5,17 +5,56 @@ * 2.0. */ -import type { SiemRuleMigrationsClient } from '../types'; - -export const createRuleMigrationClient = (): SiemRuleMigrationsClient => ({ +export const createRuleMigrationDataClient = jest.fn().mockImplementation(() => ({ create: jest.fn().mockResolvedValue({ success: true }), - search: jest.fn().mockResolvedValue([]), + getRules: jest.fn().mockResolvedValue([]), + takePending: jest.fn().mockResolvedValue([]), + saveFinished: jest.fn().mockResolvedValue({ success: true }), + saveError: jest.fn().mockResolvedValue({ success: true }), + releaseProcessing: jest.fn().mockResolvedValue({ success: true }), + releaseProcessable: jest.fn().mockResolvedValue({ success: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), +})); + +export const createRuleMigrationTaskClient = () => ({ + start: jest.fn().mockResolvedValue({ started: true }), + stop: jest.fn().mockResolvedValue({ stopped: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), }); +export const createRuleMigrationClient = () => ({ + data: createRuleMigrationDataClient(), + task: createRuleMigrationTaskClient(), +}); + +export const MockSiemRuleMigrationsClient = jest.fn().mockImplementation(createRuleMigrationClient); + export const mockSetup = jest.fn(); -export const mockGetClient = jest.fn().mockReturnValue(createRuleMigrationClient()); +export const mockCreateClient = jest.fn().mockReturnValue(createRuleMigrationClient()); +export const mockStop = jest.fn(); export const MockSiemRuleMigrationsService = jest.fn().mockImplementation(() => ({ setup: mockSetup, - getClient: mockGetClient, + createClient: mockCreateClient, + stop: mockStop, })); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts new file mode 100644 index 0000000000000..98032605ed233 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemRuleMigrationsClient } from './mocks'; +export const SiemRuleMigrationsClient = MockSiemRuleMigrationsClient; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index f4c52e9b444b8..e2505ca83beed 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -8,14 +8,11 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { v4 as uuidV4 } from 'uuid'; -import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import type { CreateRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; import { CreateRuleMigrationRequestBody } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { - SIEM_RULE_MIGRATIONS_PATH, - SiemMigrationsStatus, -} from '../../../../../common/siem_migrations/constants'; +import { SIEM_RULE_MIGRATIONS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import type { CreateRuleMigrationInput } from '../data_stream/rule_migrations_data_client'; export const registerSiemRuleMigrationsCreateRoute = ( router: SecuritySolutionPluginRouter, @@ -25,11 +22,7 @@ export const registerSiemRuleMigrationsCreateRoute = ( .post({ path: SIEM_RULE_MIGRATIONS_PATH, access: 'internal', - security: { - authz: { - requiredPrivileges: ['securitySolution'], - }, - }, + security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) .addVersion( { @@ -41,27 +34,22 @@ export const registerSiemRuleMigrationsCreateRoute = ( async (context, req, res): Promise> => { const originalRules = req.body; try { - const ctx = await context.resolve(['core', 'actions', 'securitySolution']); - - const siemMigrationClient = ctx.securitySolution.getSiemMigrationsClient(); + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); const migrationId = uuidV4(); - const timestamp = new Date().toISOString(); - const ruleMigrations = originalRules.map((originalRule) => ({ - '@timestamp': timestamp, + const ruleMigrations = originalRules.map((originalRule) => ({ migration_id: migrationId, original_rule: originalRule, - status: SiemMigrationsStatus.PENDING, })); - await siemMigrationClient.rules.create(ruleMigrations); + + await ruleMigrationsClient.data.create(ruleMigrations); return res.ok({ body: { migration_id: migrationId } }); } catch (err) { logger.error(err); - return res.badRequest({ - body: err.message, - }); + return res.badRequest({ body: err.message }); } } ); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts new file mode 100644 index 0000000000000..0efb6706918f5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { GetRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_GET_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsGetRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_GET_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(GetRuleMigrationRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const migrationRules = await ruleMigrationsClient.data.getRules(migrationId); + + return res.ok({ body: migrationRules }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index 0de49eb7df92b..f37eb2108a8a4 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -8,10 +8,20 @@ import type { Logger } from '@kbn/core/server'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { registerSiemRuleMigrationsCreateRoute } from './create'; +import { registerSiemRuleMigrationsGetRoute } from './get'; +import { registerSiemRuleMigrationsStartRoute } from './start'; +import { registerSiemRuleMigrationsStatsRoute } from './stats'; +import { registerSiemRuleMigrationsStopRoute } from './stop'; +import { registerSiemRuleMigrationsStatsAllRoute } from './stats_all'; export const registerSiemRuleMigrationsRoutes = ( router: SecuritySolutionPluginRouter, logger: Logger ) => { registerSiemRuleMigrationsCreateRoute(router, logger); + registerSiemRuleMigrationsStatsAllRoute(router, logger); + registerSiemRuleMigrationsGetRoute(router, logger); + registerSiemRuleMigrationsStartRoute(router, logger); + registerSiemRuleMigrationsStatsRoute(router, logger); + registerSiemRuleMigrationsStopRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts new file mode 100644 index 0000000000000..f97a4f2ce2398 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { APMTracer } from '@kbn/langchain/server/tracers/apm'; +import { getLangSmithTracer } from '@kbn/langchain/server/tracers/langsmith'; +import type { StartRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { + StartRuleMigrationRequestBody, + StartRuleMigrationRequestParams, +} from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_START_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStartRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATIONS_START_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(StartRuleMigrationRequestParams), + body: buildRouteValidationWithZod(StartRuleMigrationRequestBody), + }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + const { langsmith_options: langsmithOptions, connector_id: connectorId } = req.body; + + try { + const ctx = await context.resolve([ + 'core', + 'actions', + 'alerting', + 'securitySolution', + 'licensing', + ]); + if (!ctx.licensing.license.hasAtLeast('enterprise')) { + return res.forbidden({ + body: 'You must have a trial or enterprise license to use this feature', + }); + } + + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const inferenceClient = ctx.securitySolution.getInferenceClient(); + const actionsClient = ctx.actions.getActionsClient(); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + + const invocationConfig = { + callbacks: [ + new APMTracer({ projectName: langsmithOptions?.project_name ?? 'default' }, logger), + ...getLangSmithTracer({ ...langsmithOptions, logger }), + ], + }; + + const { exists, started } = await ruleMigrationsClient.task.start({ + migrationId, + connectorId, + invocationConfig, + inferenceClient, + actionsClient, + soClient, + rulesClient, + }); + + if (!exists) { + return res.noContent(); + } + return res.ok({ body: { started } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts new file mode 100644 index 0000000000000..8316e01fc6a9b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { GetRuleMigrationStatsResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationStatsRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStatsRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_STATS_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(GetRuleMigrationStatsRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const stats = await ruleMigrationsClient.task.getStats(migrationId); + + return res.ok({ body: stats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts new file mode 100644 index 0000000000000..dd2f2f503e19d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import type { GetAllStatsRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_ALL_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStatsAllRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_ALL_STATS_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { version: '1', validate: {} }, + async (context, req, res): Promise> => { + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const allStats = await ruleMigrationsClient.task.getAllStats(); + + return res.ok({ body: allStats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts new file mode 100644 index 0000000000000..4767106910186 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { StopRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { StopRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_STOP_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStopRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATIONS_STOP_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(StopRuleMigrationRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const { exists, stopped } = await ruleMigrationsClient.task.stop(migrationId); + + if (!exists) { + return res.noContent(); + } + return res.ok({ body: { stopped } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts index 103c0f9b0c952..1d9a181d2de5b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts @@ -7,9 +7,9 @@ export const mockIndexName = 'mocked_data_stream_name'; export const mockInstall = jest.fn().mockResolvedValue(undefined); -export const mockInstallSpace = jest.fn().mockResolvedValue(mockIndexName); +export const mockCreateClient = jest.fn().mockReturnValue({}); export const MockRuleMigrationsDataStream = jest.fn().mockImplementation(() => ({ install: mockInstall, - installSpace: mockInstallSpace, + createClient: mockCreateClient, })); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts new file mode 100644 index 0000000000000..83808901a0bd1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; +import assert from 'assert'; +import type { + AggregationsFilterAggregate, + AggregationsMaxAggregate, + AggregationsStringTermsAggregate, + AggregationsStringTermsBucket, + QueryDslQueryContainer, + SearchHit, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import type { StoredRuleMigration } from '../types'; +import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; +import type { + RuleMigration, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; + +export type CreateRuleMigrationInput = Omit; +export type RuleMigrationDataStats = Omit; +export type RuleMigrationAllDataStats = Array; + +export class RuleMigrationsDataClient { + constructor( + private dataStreamNamePromise: Promise, + private currentUser: AuthenticatedUser, + private esClient: ElasticsearchClient, + private logger: Logger + ) {} + + /** Indexes an array of rule migrations to be processed */ + async create(ruleMigrations: CreateRuleMigrationInput[]): Promise { + const index = await this.dataStreamNamePromise; + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: ruleMigrations.flatMap((ruleMigration) => [ + { create: { _index: index } }, + { + ...ruleMigration, + '@timestamp': new Date().toISOString(), + status: SiemMigrationStatus.PENDING, + created_by: this.currentUser.username, + }, + ]), + }) + .catch((error) => { + this.logger.error(`Error creating rule migrations: ${error.message}`); + throw error; + }); + } + + /** Retrieves an array of rule documents of a specific migrations */ + async getRules(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId); + + const storedRuleMigrations = await this.esClient + .search({ index, query, sort: '_doc' }) + .catch((error) => { + this.logger.error(`Error searching getting rule migrations: ${error.message}`); + throw error; + }) + .then((response) => this.processHits(response.hits.hits)); + return storedRuleMigrations; + } + + /** + * Retrieves `pending` rule migrations with the provided id and updates their status to `processing`. + * This operation is not atomic at migration level: + * - Multiple tasks can process different migrations simultaneously. + * - Multiple tasks should not process the same migration simultaneously. + */ + async takePending(migrationId: string, size: number): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PENDING); + + const storedRuleMigrations = await this.esClient + .search({ index, query, sort: '_doc', size }) + .catch((error) => { + this.logger.error(`Error searching for rule migrations: ${error.message}`); + throw error; + }) + .then((response) => + this.processHits(response.hits.hits, { status: SiemMigrationStatus.PROCESSING }) + ); + + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: storedRuleMigrations.flatMap(({ _id, _index, status }) => [ + { update: { _id, _index } }, + { + doc: { + status, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }, + }, + ]), + }) + .catch((error) => { + this.logger.error( + `Error updating for rule migrations status to processing: ${error.message}` + ); + throw error; + }); + + return storedRuleMigrations; + } + + /** Updates one rule migration with the provided data and sets the status to `completed` */ + async saveFinished({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + const doc = { + ...ruleMigration, + status: SiemMigrationStatus.COMPLETED, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }; + await this.esClient + .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) + .catch((error) => { + this.logger.error(`Error updating rule migration status to completed: ${error.message}`); + throw error; + }); + } + + /** Updates one rule migration with the provided data and sets the status to `failed` */ + async saveError({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + const doc = { + ...ruleMigration, + status: SiemMigrationStatus.FAILED, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }; + await this.esClient + .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) + .catch((error) => { + this.logger.error(`Error updating rule migration status to completed: ${error.message}`); + throw error; + }); + } + + /** Updates all the rule migration with the provided id with status `processing` back to `pending` */ + async releaseProcessing(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PROCESSING); + const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh: false }).catch((error) => { + this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); + throw error; + }); + } + + /** Updates all the rule migration with the provided id with status `processing` or `failed` back to `pending` */ + async releaseProcessable(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, [ + SiemMigrationStatus.PROCESSING, + SiemMigrationStatus.FAILED, + ]); + const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh: true }).catch((error) => { + this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); + throw error; + }); + } + + /** Retrieves the stats for the rule migrations with the provided id */ + async getStats(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId); + const aggregations = { + pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationStatus.PROCESSING } } }, + completed: { filter: { term: { status: SiemMigrationStatus.COMPLETED } } }, + failed: { filter: { term: { status: SiemMigrationStatus.FAILED } } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }; + const result = await this.esClient + .search({ index, query, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting rule migrations stats: ${error.message}`); + throw error; + }); + + const { pending, processing, completed, lastUpdatedAt, failed } = result.aggregations ?? {}; + return { + rules: { + total: this.getTotalHits(result), + pending: (pending as AggregationsFilterAggregate)?.doc_count ?? 0, + processing: (processing as AggregationsFilterAggregate)?.doc_count ?? 0, + completed: (completed as AggregationsFilterAggregate)?.doc_count ?? 0, + failed: (failed as AggregationsFilterAggregate)?.doc_count ?? 0, + }, + last_updated_at: (lastUpdatedAt as AggregationsMaxAggregate)?.value_as_string, + }; + } + + /** Retrieves the stats for all the rule migrations aggregated by migration id */ + async getAllStats(): Promise { + const index = await this.dataStreamNamePromise; + const aggregations = { + migrationIds: { + terms: { field: 'migration_id' }, + aggregations: { + pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationStatus.PROCESSING } } }, + completed: { filter: { term: { status: SiemMigrationStatus.COMPLETED } } }, + failed: { filter: { term: { status: SiemMigrationStatus.FAILED } } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }, + }, + }; + const result = await this.esClient + .search({ index, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting all rule migrations stats: ${error.message}`); + throw error; + }); + + const migrationsAgg = result.aggregations?.migrationIds as AggregationsStringTermsAggregate; + const buckets = (migrationsAgg?.buckets as AggregationsStringTermsBucket[]) ?? []; + return buckets.map((bucket) => ({ + migration_id: bucket.key, + rules: { + total: bucket.doc_count, + pending: bucket.pending?.doc_count ?? 0, + processing: bucket.processing?.doc_count ?? 0, + completed: bucket.completed?.doc_count ?? 0, + failed: bucket.failed?.doc_count ?? 0, + }, + last_updated_at: bucket.lastUpdatedAt?.value_as_string, + })); + } + + private getFilterQuery( + migrationId: string, + status?: SiemMigrationStatus | SiemMigrationStatus[] + ): QueryDslQueryContainer { + const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; + if (status) { + if (Array.isArray(status)) { + filter.push({ terms: { status } }); + } else { + filter.push({ term: { status } }); + } + } + return { bool: { filter } }; + } + + private processHits( + hits: Array>, + override: Partial = {} + ): StoredRuleMigration[] { + return hits.map(({ _id, _index, _source }) => { + assert(_id, 'RuleMigration document should have _id'); + assert(_source, 'RuleMigration document should have _source'); + return { ..._source, ...override, _id, _index }; + }); + } + + private getTotalHits(response: SearchResponse) { + return typeof response.hits.total === 'number' + ? response.hits.total + : response.hits.total?.value ?? 0; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts index 56510da48f1bb..467d26a380945 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts @@ -11,9 +11,19 @@ import type { InstallParams } from '@kbn/data-stream-adapter'; import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { loggerMock } from '@kbn/logging-mocks'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { securityServiceMock } from '@kbn/core-security-server-mocks'; jest.mock('@kbn/data-stream-adapter'); +// This mock is required to have a way to await the data stream name promise +const mockDataStreamNamePromise = jest.fn(); +jest.mock('./rule_migrations_data_client', () => ({ + RuleMigrationsDataClient: jest.fn((dataStreamNamePromise: Promise) => { + mockDataStreamNamePromise.mockReturnValue(dataStreamNamePromise); + }), +})); + const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass< typeof DataStreamSpacesAdapter >; @@ -21,18 +31,21 @@ const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; describe('SiemRuleMigrationsDataStream', () => { + const kibanaVersion = '8.16.0'; + const logger = loggingSystemMock.createLogger(); + beforeEach(() => { jest.clearAllMocks(); }); describe('constructor', () => { it('should create DataStreamSpacesAdapter', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1); }); it('should create component templates', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) @@ -40,7 +53,7 @@ describe('SiemRuleMigrationsDataStream', () => { }); it('should create index templates', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) @@ -50,22 +63,20 @@ describe('SiemRuleMigrationsDataStream', () => { describe('install', () => { it('should install data stream', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - const params: InstallParams = { + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + const params: Omit = { esClient, - logger: loggerMock.create(), pluginStop$: new Subject(), }; await dataStream.install(params); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(params); + expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); }); it('should log error', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - const params: InstallParams = { + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + const params: Omit = { esClient, - logger: loggerMock.create(), pluginStop$: new Subject(), }; const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; @@ -73,13 +84,16 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); await dataStream.install(params); - expect(params.logger.error).toHaveBeenCalledWith(expect.any(String), error); + expect(logger.error).toHaveBeenCalledWith(expect.any(String), error); }); }); - describe('installSpace', () => { + describe('createClient', () => { + const currentUser = securityServiceMock.createMockAuthenticatedUser(); + const createClientParams = { spaceId: 'space1', currentUser, esClient }; + it('should install space data stream', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), @@ -89,19 +103,23 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); await dataStream.install(params); - await dataStream.installSpace('space1'); + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1'); expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1'); }); it('should not install space data stream if install not executed', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - await expect(dataStream.installSpace('space1')).rejects.toThrowError(); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + await expect(async () => { + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); + }).rejects.toThrowError(); }); it('should throw error if main install had error', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), @@ -112,7 +130,10 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); await dataStream.install(params); - await expect(dataStream.installSpace('space1')).rejects.toThrowError(error); + await expect(async () => { + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); + }).rejects.toThrowError(error); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts index 83eb471e0cee3..a5855cefb1324 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts @@ -6,51 +6,69 @@ */ import { DataStreamSpacesAdapter, type InstallParams } from '@kbn/data-stream-adapter'; +import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; import { ruleMigrationsFieldMap } from './rule_migrations_field_map'; +import { RuleMigrationsDataClient } from './rule_migrations_data_client'; const TOTAL_FIELDS_LIMIT = 2500; const DATA_STREAM_NAME = '.kibana.siem-rule-migrations'; -const ECS_COMPONENT_TEMPLATE_NAME = 'ecs'; + +interface RuleMigrationsDataStreamCreateClientParams { + spaceId: string; + currentUser: AuthenticatedUser; + esClient: ElasticsearchClient; +} export class RuleMigrationsDataStream { - private readonly dataStream: DataStreamSpacesAdapter; + private readonly dataStreamAdapter: DataStreamSpacesAdapter; private installPromise?: Promise; - constructor({ kibanaVersion }: { kibanaVersion: string }) { - this.dataStream = new DataStreamSpacesAdapter(DATA_STREAM_NAME, { + constructor(private logger: Logger, kibanaVersion: string) { + this.dataStreamAdapter = new DataStreamSpacesAdapter(DATA_STREAM_NAME, { kibanaVersion, totalFieldsLimit: TOTAL_FIELDS_LIMIT, }); - this.dataStream.setComponentTemplate({ + this.dataStreamAdapter.setComponentTemplate({ name: DATA_STREAM_NAME, fieldMap: ruleMigrationsFieldMap, }); - this.dataStream.setIndexTemplate({ + this.dataStreamAdapter.setIndexTemplate({ name: DATA_STREAM_NAME, - componentTemplateRefs: [DATA_STREAM_NAME, ECS_COMPONENT_TEMPLATE_NAME], + componentTemplateRefs: [DATA_STREAM_NAME], }); } - async install(params: InstallParams) { + async install(params: Omit) { try { - this.installPromise = this.dataStream.install(params); + this.installPromise = this.dataStreamAdapter.install({ ...params, logger: this.logger }); await this.installPromise; } catch (err) { - params.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err); + this.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err); } } - async installSpace(spaceId: string): Promise { + createClient({ + spaceId, + currentUser, + esClient, + }: RuleMigrationsDataStreamCreateClientParams): RuleMigrationsDataClient { + const dataStreamNamePromise = this.installSpace(spaceId); + return new RuleMigrationsDataClient(dataStreamNamePromise, currentUser, esClient, this.logger); + } + + // Installs the data stream for the specific space. it will only install if it hasn't been installed yet. + // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed. + private async installSpace(spaceId: string): Promise { if (!this.installPromise) { throw new Error('Siem rule migrations data stream not installed'); } // wait for install to complete, may reject if install failed, routes should handle this await this.installPromise; - let dataStreamName = await this.dataStream.getInstalledSpaceName(spaceId); + let dataStreamName = await this.dataStreamAdapter.getInstalledSpaceName(spaceId); if (!dataStreamName) { - dataStreamName = await this.dataStream.installSpace(spaceId); + dataStreamName = await this.dataStreamAdapter.installSpace(spaceId); } return dataStreamName; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts index ba9a706957bcb..a65cd45b832e9 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts @@ -11,6 +11,7 @@ import type { RuleMigration } from '../../../../../common/siem_migrations/model/ export const ruleMigrationsFieldMap: FieldMap> = { '@timestamp': { type: 'date', required: false }, migration_id: { type: 'keyword', required: true }, + created_by: { type: 'keyword', required: true }, status: { type: 'keyword', required: true }, original_rule: { type: 'nested', required: true }, 'original_rule.vendor': { type: 'keyword', required: true }, @@ -28,7 +29,7 @@ export const ruleMigrationsFieldMap: FieldMap> 'elastic_rule.severity': { type: 'keyword', required: false }, 'elastic_rule.prebuilt_rule_id': { type: 'keyword', required: false }, 'elastic_rule.id': { type: 'keyword', required: false }, - translation_state: { type: 'keyword', required: false }, + translation_result: { type: 'keyword', required: false }, comments: { type: 'text', array: true, required: false }, updated_at: { type: 'date', required: false }, updated_by: { type: 'keyword', required: false }, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts index 390d302264cea..5c611d85e0464 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts @@ -8,25 +8,28 @@ import { loggingSystemMock, elasticsearchServiceMock, httpServerMock, + securityServiceMock, } from '@kbn/core/server/mocks'; import { SiemRuleMigrationsService } from './siem_rule_migrations_service'; import { Subject } from 'rxjs'; -import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen'; import { MockRuleMigrationsDataStream, mockInstall, - mockInstallSpace, - mockIndexName, + mockCreateClient, } from './data_stream/__mocks__/mocks'; -import type { KibanaRequest } from '@kbn/core/server'; +import type { SiemRuleMigrationsCreateClientParams } from './types'; jest.mock('./data_stream/rule_migrations_data_stream'); +jest.mock('./task/rule_migrations_task_runner', () => ({ + RuleMigrationsTaskRunner: jest.fn(), +})); describe('SiemRuleMigrationsService', () => { let ruleMigrationsService: SiemRuleMigrationsService; const kibanaVersion = '8.16.0'; const esClusterClient = elasticsearchServiceMock.createClusterClient(); + const currentUser = securityServiceMock.createMockAuthenticatedUser(); const logger = loggingSystemMock.createLogger(); const pluginStop$ = new Subject(); @@ -36,7 +39,7 @@ describe('SiemRuleMigrationsService', () => { }); it('should instantiate the rule migrations data stream adapter', () => { - expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith({ kibanaVersion }); + expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith(logger, kibanaVersion); }); describe('when setup is called', () => { @@ -45,22 +48,26 @@ describe('SiemRuleMigrationsService', () => { expect(mockInstall).toHaveBeenCalledWith({ esClient: esClusterClient.asInternalUser, - logger, pluginStop$, }); }); }); - describe('when getClient is called', () => { - let request: KibanaRequest; + describe('when createClient is called', () => { + let createClientParams: SiemRuleMigrationsCreateClientParams; + beforeEach(() => { - request = httpServerMock.createKibanaRequest(); + createClientParams = { + spaceId: 'default', + currentUser, + request: httpServerMock.createKibanaRequest(), + }; }); describe('without setup', () => { it('should throw an error', () => { expect(() => { - ruleMigrationsService.getClient({ spaceId: 'default', request }); + ruleMigrationsService.createClient(createClientParams); }).toThrowError('ES client not available, please call setup first'); }); }); @@ -71,44 +78,19 @@ describe('SiemRuleMigrationsService', () => { }); it('should call installSpace', () => { - ruleMigrationsService.getClient({ spaceId: 'default', request }); - - expect(mockInstallSpace).toHaveBeenCalledWith('default'); - }); - - it('should return a client with create and search methods after setup', () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); - - expect(client).toHaveProperty('create'); - expect(client).toHaveProperty('search'); + ruleMigrationsService.createClient(createClientParams); + expect(mockCreateClient).toHaveBeenCalledWith({ + spaceId: createClientParams.spaceId, + currentUser: createClientParams.currentUser, + esClient: esClusterClient.asScoped().asCurrentUser, + }); }); - it('should call ES bulk create API with the correct parameters with create is called', async () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); - - const ruleMigrations = [{ migration_id: 'dummy_migration_id' } as RuleMigration]; - await client.create(ruleMigrations); - - expect(esClusterClient.asScoped().asCurrentUser.bulk).toHaveBeenCalledWith( - expect.objectContaining({ - body: [{ create: { _index: mockIndexName } }, { migration_id: 'dummy_migration_id' }], - refresh: 'wait_for', - }) - ); - }); - - it('should call ES search API with the correct parameters with search is called', async () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); - - const term = { migration_id: 'dummy_migration_id' }; - await client.search(term); + it('should return data and task clients', () => { + const client = ruleMigrationsService.createClient(createClientParams); - expect(esClusterClient.asScoped().asCurrentUser.search).toHaveBeenCalledWith( - expect.objectContaining({ - index: mockIndexName, - body: { query: { term } }, - }) - ); + expect(client).toHaveProperty('data'); + expect(client).toHaveProperty('task'); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index 5b20f957cb6fa..1bf9dcf11fd95 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -5,52 +5,67 @@ * 2.0. */ +import assert from 'assert'; import type { IClusterClient, Logger } from '@kbn/core/server'; import { RuleMigrationsDataStream } from './data_stream/rule_migrations_data_stream'; import type { - SiemRuleMigrationsClient, SiemRulesMigrationsSetupParams, - SiemRuleMigrationsGetClientParams, + SiemRuleMigrationsCreateClientParams, + SiemRuleMigrationsClient, } from './types'; +import { RuleMigrationsTaskRunner } from './task/rule_migrations_task_runner'; export class SiemRuleMigrationsService { - private dataStreamAdapter: RuleMigrationsDataStream; + private rulesDataStream: RuleMigrationsDataStream; private esClusterClient?: IClusterClient; + private taskRunner: RuleMigrationsTaskRunner; constructor(private logger: Logger, kibanaVersion: string) { - this.dataStreamAdapter = new RuleMigrationsDataStream({ kibanaVersion }); + this.rulesDataStream = new RuleMigrationsDataStream(this.logger, kibanaVersion); + this.taskRunner = new RuleMigrationsTaskRunner(this.logger); } setup({ esClusterClient, ...params }: SiemRulesMigrationsSetupParams) { this.esClusterClient = esClusterClient; const esClient = esClusterClient.asInternalUser; - this.dataStreamAdapter.install({ ...params, esClient, logger: this.logger }).catch((err) => { + + this.rulesDataStream.install({ ...params, esClient }).catch((err) => { this.logger.error(`Error installing data stream for rule migrations: ${err.message}`); throw err; }); } - getClient({ spaceId, request }: SiemRuleMigrationsGetClientParams): SiemRuleMigrationsClient { - if (!this.esClusterClient) { - throw new Error('ES client not available, please call setup first'); - } - // Installs the data stream for the specific space. it will only install if it hasn't been installed yet. - // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed. - const dataStreamNamePromise = this.dataStreamAdapter.installSpace(spaceId); + createClient({ + spaceId, + currentUser, + request, + }: SiemRuleMigrationsCreateClientParams): SiemRuleMigrationsClient { + assert(currentUser, 'Current user must be authenticated'); + assert(this.esClusterClient, 'ES client not available, please call setup first'); const esClient = this.esClusterClient.asScoped(request).asCurrentUser; + const dataClient = this.rulesDataStream.createClient({ spaceId, currentUser, esClient }); + return { - create: async (ruleMigrations) => { - const _index = await dataStreamNamePromise; - return esClient.bulk({ - refresh: 'wait_for', - body: ruleMigrations.flatMap((ruleMigration) => [{ create: { _index } }, ruleMigration]), - }); - }, - search: async (term) => { - const index = await dataStreamNamePromise; - return esClient.search({ index, body: { query: { term } } }); + data: dataClient, + task: { + start: (params) => { + return this.taskRunner.start({ ...params, currentUser, dataClient }); + }, + stop: (migrationId) => { + return this.taskRunner.stop({ migrationId, dataClient }); + }, + getStats: async (migrationId) => { + return this.taskRunner.getStats({ migrationId, dataClient }); + }, + getAllStats: async () => { + return this.taskRunner.getAllStats({ dataClient }); + }, }, }; } + + stop() { + this.taskRunner.stopAll(); + } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts new file mode 100644 index 0000000000000..a44197d82850f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { END, START, StateGraph } from '@langchain/langgraph'; +import { migrateRuleState } from './state'; +import type { MigrateRuleGraphParams, MigrateRuleState } from './types'; +import { getTranslateQueryNode } from './nodes/translate_query'; +import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule'; + +export function getRuleMigrationAgent({ + model, + inferenceClient, + prebuiltRulesMap, + connectorId, + logger, +}: MigrateRuleGraphParams) { + const matchPrebuiltRuleNode = getMatchPrebuiltRuleNode({ model, prebuiltRulesMap, logger }); + const translationNode = getTranslateQueryNode({ inferenceClient, connectorId, logger }); + + const translateRuleGraph = new StateGraph(migrateRuleState) + // Nodes + .addNode('matchPrebuiltRule', matchPrebuiltRuleNode) + .addNode('translation', translationNode) + // Edges + .addEdge(START, 'matchPrebuiltRule') + .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional) + .addEdge('translation', END); + + const graph = translateRuleGraph.compile(); + graph.name = 'Rule Migration Graph'; // Customizes the name displayed in LangSmith + return graph; +} + +const matchedPrebuiltRuleConditional = (state: MigrateRuleState) => { + if (state.elastic_rule?.prebuilt_rule_id) { + return END; + } + return 'translation'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts new file mode 100644 index 0000000000000..febf5fc85f5a0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getRuleMigrationAgent } from './graph'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts new file mode 100644 index 0000000000000..2d8b81d00eafb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getMatchPrebuiltRuleNode } from './match_prebuilt_rule'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts new file mode 100644 index 0000000000000..4a0404acf653d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import type { ChatModel } from '../../../util/actions_client_chat'; +import type { GraphNode } from '../../types'; +import { filterPrebuiltRules, type PrebuiltRulesMapByName } from '../../../util/prebuilt_rules'; +import { MATCH_PREBUILT_RULE_PROMPT } from './prompts'; + +interface GetMatchPrebuiltRuleNodeParams { + model: ChatModel; + prebuiltRulesMap: PrebuiltRulesMapByName; + logger: Logger; +} + +export const getMatchPrebuiltRuleNode = + ({ model, prebuiltRulesMap }: GetMatchPrebuiltRuleNodeParams): GraphNode => + async (state) => { + const mitreAttackIds = state.original_rule.mitre_attack_ids; + if (!mitreAttackIds?.length) { + return {}; + } + const filteredPrebuiltRulesMap = filterPrebuiltRules(prebuiltRulesMap, mitreAttackIds); + if (filteredPrebuiltRulesMap.size === 0) { + return {}; + } + + const outputParser = new StringOutputParser(); + const matchPrebuiltRule = MATCH_PREBUILT_RULE_PROMPT.pipe(model).pipe(outputParser); + + const elasticSecurityRules = Array(filteredPrebuiltRulesMap.keys()).join('\n'); + const response = await matchPrebuiltRule.invoke({ + elasticSecurityRules, + ruleTitle: state.original_rule.title, + }); + const cleanResponse = response.trim(); + if (cleanResponse === 'no_match') { + return {}; + } + + const result = filteredPrebuiltRulesMap.get(cleanResponse); + if (result != null) { + return { + elastic_rule: { + title: result.rule.name, + description: result.rule.description, + prebuilt_rule_id: result.rule.rule_id, + id: result.installedRuleId, + }, + }; + } + + return {}; + }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts new file mode 100644 index 0000000000000..434636d0519b1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatPromptTemplate } from '@langchain/core/prompts'; +export const MATCH_PREBUILT_RULE_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are an expert assistant in Cybersecurity, your task is to help migrating a SIEM detection rule, from Splunk Security to Elastic Security. +You will be provided with a Splunk Detection Rule name by the user, your goal is to try find an Elastic Detection Rule that covers the same threat, if any. +The list of Elastic Detection Rules suggested is provided in the context below. + +Guidelines: +If there is no Elastic rule in the list that covers the same threat, answer only with the string: no_match +If there is one Elastic rule in the list that covers the same threat, answer only with its name without any further explanation. +If there are multiple rules in the list that cover the same threat, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". + + +{elasticSecurityRules} + +`, + ], + [ + 'human', + `The Splunk Detection Rule is: +<> +{ruleTitle} +<> +`, + ], + ['ai', 'Please find the answer below:'], +]); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts new file mode 100644 index 0000000000000..2277f2fae41a9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { naturalLanguageToEsql, type InferenceClient } from '@kbn/inference-plugin/server'; +import { lastValueFrom } from 'rxjs'; + +export type EsqlKnowledgeBaseCaller = (input: string) => Promise; + +type GetEsqlTranslatorToolParams = (params: { + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +}) => EsqlKnowledgeBaseCaller; + +export const getEsqlKnowledgeBase: GetEsqlTranslatorToolParams = + ({ inferenceClient: client, connectorId, logger }) => + async (input: string) => { + const { content } = await lastValueFrom( + naturalLanguageToEsql({ + client, + connectorId, + input, + logger: { + debug: (source) => { + logger.debug(typeof source === 'function' ? source() : source); + }, + }, + }) + ); + return content; + }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts new file mode 100644 index 0000000000000..7d247f755e9da --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getTranslateQueryNode } from './translate_query'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts new file mode 100644 index 0000000000000..0b97faf7dc96f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MigrateRuleState } from '../../types'; + +export const getEsqlTranslationPrompt = ( + state: MigrateRuleState +): string => `You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk to Elastic Security. +Below you will find Splunk rule information: the title, description and the SPL (Search Processing Language) query. +Your goal is to translate the SPL query into an equivalent Elastic Security Query Language (ES|QL) query. + +Guidelines: +- Start the translation process by analyzing the SPL query and identifying the key components. +- Always use logs* index pattern for the ES|QL translated query. +- If, in the SPL query, you find a lookup list or macro that, based only on its name, you can not translate with confidence to ES|QL, mention it in the summary and +add a placeholder in the query with the format [macro:(parameters)] or [lookup:] including the [] keys, example: [macro:my_macro(first_param,second_param)] or [lookup:my_lookup]. + +The output will be parsed and should contain: +- First, the ES|QL query inside an \`\`\`esql code block. +- At the end, the summary of the translation process followed in markdown, starting with "## Migration Summary". + +This is the Splunk rule information: + +<> +${state.original_rule.title} +<> + +<> +${state.original_rule.description} +<> + +<> +${state.original_rule.query} +<> +`; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts new file mode 100644 index 0000000000000..00e1e60c7b5f3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { GraphNode } from '../../types'; +import { getEsqlKnowledgeBase } from './esql_knowledge_base_caller'; +import { getEsqlTranslationPrompt } from './prompt'; +import { SiemMigrationRuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; + +interface GetTranslateQueryNodeParams { + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +} + +export const getTranslateQueryNode = ({ + inferenceClient, + connectorId, + logger, +}: GetTranslateQueryNodeParams): GraphNode => { + const esqlKnowledgeBaseCaller = getEsqlKnowledgeBase({ inferenceClient, connectorId, logger }); + return async (state) => { + const input = getEsqlTranslationPrompt(state); + const response = await esqlKnowledgeBaseCaller(input); + + const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; + const summary = response.match(/## Migration Summary[\s\S]*$/)?.[0] ?? ''; + + const translationResult = getTranslationResult(esqlQuery); + + return { + response, + comments: [summary], + translation_result: translationResult, + elastic_rule: { + title: state.original_rule.title, + description: state.original_rule.description, + severity: 'low', + query: esqlQuery, + query_language: 'esql', + }, + }; + }; +}; + +const getTranslationResult = (esqlQuery: string): SiemMigrationRuleTranslationResult => { + if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { + return SiemMigrationRuleTranslationResult.PARTIAL; + } + return SiemMigrationRuleTranslationResult.FULL; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts new file mode 100644 index 0000000000000..c1e510bdc052d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BaseMessage } from '@langchain/core/messages'; +import { Annotation, messagesStateReducer } from '@langchain/langgraph'; +import type { + ElasticRule, + OriginalRule, + RuleMigration, +} from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { SiemMigrationRuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; + +export const migrateRuleState = Annotation.Root({ + messages: Annotation({ + reducer: messagesStateReducer, + default: () => [], + }), + original_rule: Annotation(), + elastic_rule: Annotation({ + reducer: (state, action) => ({ ...state, ...action }), + }), + translation_result: Annotation(), + comments: Annotation({ + reducer: (current, value) => (value ? (current ?? []).concat(value) : current), + default: () => [], + }), + response: Annotation(), +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts new file mode 100644 index 0000000000000..643d200e4b0bf --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { migrateRuleState } from './state'; +import type { ChatModel } from '../util/actions_client_chat'; +import type { PrebuiltRulesMapByName } from '../util/prebuilt_rules'; + +export type MigrateRuleState = typeof migrateRuleState.State; +export type GraphNode = (state: MigrateRuleState) => Promise>; + +export interface MigrateRuleGraphParams { + inferenceClient: InferenceClient; + model: ChatModel; + connectorId: string; + prebuiltRulesMap: PrebuiltRulesMapByName; + logger: Logger; +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts new file mode 100644 index 0000000000000..6ae7294fb5257 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { AbortError, abortSignalToPromise } from '@kbn/kibana-utils-plugin/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { + RuleMigrationAllTaskStats, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationDataStats } from '../data_stream/rule_migrations_data_client'; +import type { + RuleMigrationTaskStartParams, + RuleMigrationTaskStartResult, + RuleMigrationTaskStatsParams, + RuleMigrationTaskStopParams, + RuleMigrationTaskStopResult, + RuleMigrationTaskPrepareParams, + RuleMigrationTaskRunParams, + MigrationAgent, + RuleMigrationAllTaskStatsParams, +} from './types'; +import { getRuleMigrationAgent } from './agent'; +import type { MigrateRuleState } from './agent/types'; +import { retrievePrebuiltRulesMap } from './util/prebuilt_rules'; +import { ActionsClientChat } from './util/actions_client_chat'; + +interface TaskLogger { + info: (msg: string) => void; + debug: (msg: string) => void; + error: (msg: string, error: Error) => void; +} +const getTaskLogger = (logger: Logger): TaskLogger => { + const prefix = '[ruleMigrationsTask]: '; + return { + info: (msg) => logger.info(`${prefix}${msg}`), + debug: (msg) => logger.debug(`${prefix}${msg}`), + error: (msg, error) => logger.error(`${prefix}${msg}: ${error.message}`), + }; +}; + +const ITERATION_BATCH_SIZE = 50 as const; +const ITERATION_SLEEP_SECONDS = 10 as const; + +export class RuleMigrationsTaskRunner { + private migrationsRunning: Map; + private taskLogger: TaskLogger; + + constructor(private logger: Logger) { + this.migrationsRunning = new Map(); + this.taskLogger = getTaskLogger(logger); + } + + /** Starts a rule migration task */ + async start(params: RuleMigrationTaskStartParams): Promise { + const { migrationId, dataClient } = params; + if (this.migrationsRunning.has(migrationId)) { + return { exists: true, started: false }; + } + // Just in case some previous execution was interrupted without releasing + await dataClient.releaseProcessable(migrationId); + + const { rules } = await dataClient.getStats(migrationId); + if (rules.total === 0) { + return { exists: false, started: false }; + } + if (rules.pending === 0) { + return { exists: true, started: false }; + } + + const abortController = new AbortController(); + + // Await the preparation to make sure the agent is created properly so the task can run + const agent = await this.prepare({ ...params, abortController }); + + // not awaiting the `run` promise to execute the task in the background + this.run({ ...params, agent, abortController }).catch((err) => { + // All errors in the `run` method are already catch, this should never happen, but just in case + this.taskLogger.error(`Unexpected error running the migration ID:${migrationId}`, err); + }); + + return { exists: true, started: true }; + } + + private async prepare({ + connectorId, + inferenceClient, + actionsClient, + rulesClient, + soClient, + abortController, + }: RuleMigrationTaskPrepareParams): Promise { + const prebuiltRulesMap = await retrievePrebuiltRulesMap({ soClient, rulesClient }); + + const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger); + const model = await actionsClientChat.createModel({ + signal: abortController.signal, + temperature: 0.05, + }); + + const agent = getRuleMigrationAgent({ + connectorId, + model, + inferenceClient, + prebuiltRulesMap, + logger: this.logger, + }); + return agent; + } + + private async run({ + migrationId, + agent, + dataClient, + currentUser, + invocationConfig, + abortController, + }: RuleMigrationTaskRunParams): Promise { + if (this.migrationsRunning.has(migrationId)) { + // This should never happen, but just in case + throw new Error(`Task already running for migration ID:${migrationId} `); + } + this.taskLogger.info(`Starting migration ID:${migrationId}`); + + this.migrationsRunning.set(migrationId, { user: currentUser.username, abortController }); + const config: RunnableConfig = { + ...invocationConfig, + // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319 + }; + + const abortPromise = abortSignalToPromise(abortController.signal); + + try { + const sleep = async (seconds: number) => { + this.taskLogger.debug(`Sleeping ${seconds}s for migration ID:${migrationId}`); + await Promise.race([ + new Promise((resolve) => setTimeout(resolve, seconds * 1000)), + abortPromise.promise, + ]); + }; + + let isDone: boolean = false; + do { + const ruleMigrations = await dataClient.takePending(migrationId, ITERATION_BATCH_SIZE); + this.taskLogger.debug( + `Processing ${ruleMigrations.length} rules for migration ID:${migrationId}` + ); + + await Promise.all( + ruleMigrations.map(async (ruleMigration) => { + this.taskLogger.debug( + `Starting migration of rule "${ruleMigration.original_rule.title}"` + ); + try { + const start = Date.now(); + + const ruleMigrationResult: MigrateRuleState = await Promise.race([ + agent.invoke({ original_rule: ruleMigration.original_rule }, config), + abortPromise.promise, // workaround for the issue with the langGraph signal + ]); + + const duration = (Date.now() - start) / 1000; + this.taskLogger.debug( + `Migration of rule "${ruleMigration.original_rule.title}" finished in ${duration}s` + ); + + await dataClient.saveFinished({ + ...ruleMigration, + elastic_rule: ruleMigrationResult.elastic_rule, + translation_result: ruleMigrationResult.translation_result, + comments: ruleMigrationResult.comments, + }); + } catch (error) { + if (error instanceof AbortError) { + throw error; + } + this.taskLogger.error( + `Error migrating rule "${ruleMigration.original_rule.title}"`, + error + ); + await dataClient.saveError({ + ...ruleMigration, + comments: [`Error migrating rule: ${error.message}`], + }); + } + }) + ); + + this.taskLogger.debug(`Batch processed successfully for migration ID:${migrationId}`); + + const { rules } = await dataClient.getStats(migrationId); + isDone = rules.pending === 0; + if (!isDone) { + await sleep(ITERATION_SLEEP_SECONDS); + } + } while (!isDone); + + this.taskLogger.info(`Finished migration ID:${migrationId}`); + } catch (error) { + await dataClient.releaseProcessing(migrationId); + + if (error instanceof AbortError) { + this.taskLogger.info(`Abort signal received, stopping migration ID:${migrationId}`); + return; + } else { + this.taskLogger.error(`Error processing migration ID:${migrationId}`, error); + } + } finally { + this.migrationsRunning.delete(migrationId); + abortPromise.cleanup(); + } + } + + /** Returns the stats of a migration */ + async getStats({ + migrationId, + dataClient, + }: RuleMigrationTaskStatsParams): Promise { + const dataStats = await dataClient.getStats(migrationId); + const status = this.getTaskStatus(migrationId, dataStats.rules); + return { status, ...dataStats }; + } + + /** Returns the stats of all migrations */ + async getAllStats({ + dataClient, + }: RuleMigrationAllTaskStatsParams): Promise { + const allDataStats = await dataClient.getAllStats(); + return allDataStats.map((dataStats) => { + const status = this.getTaskStatus(dataStats.migration_id, dataStats.rules); + return { status, ...dataStats }; + }); + } + + private getTaskStatus( + migrationId: string, + dataStats: RuleMigrationDataStats['rules'] + ): RuleMigrationTaskStats['status'] { + if (this.migrationsRunning.has(migrationId)) { + return 'running'; + } + if (dataStats.pending === dataStats.total) { + return 'ready'; + } + if (dataStats.completed + dataStats.failed === dataStats.total) { + return 'finished'; + } + return 'stopped'; + } + + /** Stops one running migration */ + async stop({ + migrationId, + dataClient, + }: RuleMigrationTaskStopParams): Promise { + try { + const migrationRunning = this.migrationsRunning.get(migrationId); + if (migrationRunning) { + migrationRunning.abortController.abort(); + return { exists: true, stopped: true }; + } + + const { rules } = await dataClient.getStats(migrationId); + if (rules.total > 0) { + return { exists: true, stopped: false }; + } + return { exists: false, stopped: false }; + } catch (err) { + this.taskLogger.error(`Error stopping migration ID:${migrationId}`, err); + return { exists: true, stopped: false }; + } + } + + /** Stops all running migrations */ + stopAll() { + this.migrationsRunning.forEach((migrationRunning) => { + migrationRunning.abortController.abort(); + }); + this.migrationsRunning.clear(); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts new file mode 100644 index 0000000000000..e26a5b7216f48 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, SavedObjectsClientContract } from '@kbn/core/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { RuleMigrationsDataClient } from '../data_stream/rule_migrations_data_client'; +import type { getRuleMigrationAgent } from './agent'; + +export type MigrationAgent = ReturnType; + +export interface RuleMigrationTaskStartParams { + migrationId: string; + currentUser: AuthenticatedUser; + connectorId: string; + invocationConfig: RunnableConfig; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskPrepareParams { + connectorId: string; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; + abortController: AbortController; +} + +export interface RuleMigrationTaskRunParams { + migrationId: string; + currentUser: AuthenticatedUser; + invocationConfig: RunnableConfig; + agent: MigrationAgent; + dataClient: RuleMigrationsDataClient; + abortController: AbortController; +} + +export interface RuleMigrationTaskStopParams { + migrationId: string; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskStatsParams { + migrationId: string; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationAllTaskStatsParams { + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskStartResult { + started: boolean; + exists: boolean; +} + +export interface RuleMigrationTaskStopResult { + stopped: boolean; + exists: boolean; +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts new file mode 100644 index 0000000000000..204978c901df6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ActionsClientSimpleChatModel } from '@kbn/langchain/server'; +import { + ActionsClientBedrockChatModel, + ActionsClientChatOpenAI, + ActionsClientChatVertexAI, +} from '@kbn/langchain/server'; +import type { Logger } from '@kbn/core/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { ActionsClientChatOpenAIParams } from '@kbn/langchain/server/language_models/chat_openai'; +import type { CustomChatModelInput as ActionsClientBedrockChatModelParams } from '@kbn/langchain/server/language_models/bedrock_chat'; +import type { CustomChatModelInput as ActionsClientChatVertexAIParams } from '@kbn/langchain/server/language_models/gemini_chat'; +import type { CustomChatModelInput as ActionsClientSimpleChatModelParams } from '@kbn/langchain/server/language_models/simple_chat_model'; + +export type ChatModel = + | ActionsClientSimpleChatModel + | ActionsClientChatOpenAI + | ActionsClientBedrockChatModel + | ActionsClientChatVertexAI; + +export type ActionsClientChatModelClass = + | typeof ActionsClientSimpleChatModel + | typeof ActionsClientChatOpenAI + | typeof ActionsClientBedrockChatModel + | typeof ActionsClientChatVertexAI; + +export type ChatModelParams = Partial & + Partial & + Partial & + Partial & { + /** Enables the streaming mode of the response, disabled by default */ + streaming?: boolean; + }; + +const llmTypeDictionary: Record = { + [`.gen-ai`]: `openai`, + [`.bedrock`]: `bedrock`, + [`.gemini`]: `gemini`, +}; + +export class ActionsClientChat { + constructor( + private readonly connectorId: string, + private readonly actionsClient: ActionsClient, + private readonly logger: Logger + ) {} + + public async createModel(params?: ChatModelParams): Promise { + const connector = await this.actionsClient.get({ id: this.connectorId }); + if (!connector) { + throw new Error(`Connector not found: ${this.connectorId}`); + } + + const llmType = this.getLLMType(connector.actionTypeId); + const ChatModelClass = this.getLLMClass(llmType); + + const model = new ChatModelClass({ + actionsClient: this.actionsClient, + connectorId: this.connectorId, + logger: this.logger, + llmType, + model: connector.config?.defaultModel, + ...params, + streaming: params?.streaming ?? false, // disabling streaming by default, for some reason is enabled when omitted + }); + return model; + } + + private getLLMType(actionTypeId: string): string | undefined { + if (llmTypeDictionary[actionTypeId]) { + return llmTypeDictionary[actionTypeId]; + } + throw new Error(`Unknown LLM type for action type ID: ${actionTypeId}`); + } + + private getLLMClass(llmType?: string): ActionsClientChatModelClass { + switch (llmType) { + case 'bedrock': + return ActionsClientBedrockChatModel; + case 'gemini': + return ActionsClientChatVertexAI; + case 'openai': + default: + return ActionsClientChatOpenAI; + } + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts new file mode 100644 index 0000000000000..55256d0ad0fdd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; +import type { PrebuiltRulesMapByName } from './prebuilt_rules'; +import { filterPrebuiltRules, retrievePrebuiltRulesMap } from './prebuilt_rules'; +import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; + +jest.mock( + '../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client', + () => ({ createPrebuiltRuleObjectsClient: jest.fn() }) +); +jest.mock( + '../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client', + () => ({ createPrebuiltRuleAssetsClient: jest.fn() }) +); + +const mitreAttackIds = 'T1234'; +const rule1 = { + name: 'rule one', + id: 'rule1', + threat: [ + { + framework: 'MITRE ATT&CK', + technique: [{ id: mitreAttackIds, name: 'tactic one' }], + }, + ], +}; +const rule2 = { + name: 'rule two', + id: 'rule2', +}; + +const defaultRuleVersionsTriad = new Map([ + ['rule1', { target: rule1 }], + ['rule2', { target: rule2, current: rule2 }], +]); +const mockFetchRuleVersionsTriad = jest.fn().mockResolvedValue(defaultRuleVersionsTriad); +jest.mock( + '../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad', + () => ({ + fetchRuleVersionsTriad: () => mockFetchRuleVersionsTriad(), + }) +); + +const defaultParams = { + soClient: savedObjectsClientMock.create(), + rulesClient: rulesClientMock.create(), +}; + +describe('retrievePrebuiltRulesMap', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('when prebuilt rule is installed', () => { + it('should return isInstalled flag', async () => { + const prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); + expect(prebuiltRulesMap.size).toBe(2); + expect(prebuiltRulesMap.get('rule one')).toEqual( + expect.objectContaining({ installedRuleId: undefined }) + ); + expect(prebuiltRulesMap.get('rule two')).toEqual( + expect.objectContaining({ installedRuleId: rule2.id }) + ); + }); + }); +}); + +describe('filterPrebuiltRules', () => { + let prebuiltRulesMap: PrebuiltRulesMapByName; + + beforeEach(async () => { + prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); + jest.clearAllMocks(); + }); + + describe('when splunk rule contains empty mitreAttackIds', () => { + it('should return empty rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, []); + expect(filteredPrebuiltRules.size).toBe(0); + }); + }); + + describe('when splunk rule does not match mitreAttackIds', () => { + it('should return empty rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, [`${mitreAttackIds}_2`]); + expect(filteredPrebuiltRules.size).toBe(0); + }); + }); + + describe('when splunk rule contains matching mitreAttackIds', () => { + it('should return the filtered rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, [mitreAttackIds]); + expect(filteredPrebuiltRules.size).toBe(1); + expect(filteredPrebuiltRules.get('rule one')).toEqual( + expect.objectContaining({ rule: rule1 }) + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts new file mode 100644 index 0000000000000..ade6632aaa5b5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import type { PrebuiltRuleAsset } from '../../../../detection_engine/prebuilt_rules'; +import { fetchRuleVersionsTriad } from '../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad'; +import { createPrebuiltRuleObjectsClient } from '../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client'; +import { createPrebuiltRuleAssetsClient } from '../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; + +export interface PrebuiltRuleMapped { + rule: PrebuiltRuleAsset; + installedRuleId?: string; +} + +export type PrebuiltRulesMapByName = Map; + +interface RetrievePrebuiltRulesParams { + soClient: SavedObjectsClientContract; + rulesClient: RulesClient; +} + +export const retrievePrebuiltRulesMap = async ({ + soClient, + rulesClient, +}: RetrievePrebuiltRulesParams): Promise => { + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); + + const prebuiltRulesMap = await fetchRuleVersionsTriad({ + ruleAssetsClient, + ruleObjectsClient, + }); + const prebuiltRulesByName: PrebuiltRulesMapByName = new Map(); + prebuiltRulesMap.forEach((ruleVersions) => { + const rule = ruleVersions.target || ruleVersions.current; + if (rule) { + prebuiltRulesByName.set(rule.name, { + rule, + installedRuleId: ruleVersions.current?.id, + }); + } + }); + return prebuiltRulesByName; +}; + +export const filterPrebuiltRules = ( + prebuiltRulesByName: PrebuiltRulesMapByName, + mitreAttackIds: string[] +) => { + const filteredPrebuiltRulesByName = new Map(); + if (mitreAttackIds?.length) { + // If this rule has MITRE ATT&CK IDs, remove unrelated prebuilt rules + prebuiltRulesByName.forEach(({ rule }, ruleName) => { + const mitreAttackThreat = rule.threat?.filter( + ({ framework }) => framework === 'MITRE ATT&CK' + ); + if (!mitreAttackThreat) { + // If this rule has no MITRE ATT&CK reference we skip it + return; + } + + const sameTechnique = mitreAttackThreat.find((threat) => + threat.technique?.some(({ id }) => mitreAttackIds?.includes(id)) + ); + + if (sameTechnique) { + filteredPrebuiltRulesByName.set(ruleName, prebuiltRulesByName.get(ruleName)); + } + }); + } + return filteredPrebuiltRulesByName; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index 1892032a21723..78ec2ef89c7a3 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -5,10 +5,29 @@ * 2.0. */ -import type { BulkResponse, SearchResponse } from '@elastic/elasticsearch/lib/api/types'; -import type { IClusterClient, KibanaRequest } from '@kbn/core/server'; +import type { + AuthenticatedUser, + IClusterClient, + KibanaRequest, + SavedObjectsClientContract, +} from '@kbn/core/server'; import type { Subject } from 'rxjs'; -import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { + RuleMigration, + RuleMigrationAllTaskStats, + RuleMigrationTaskStats, +} from '../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationsDataClient } from './data_stream/rule_migrations_data_client'; +import type { RuleMigrationTaskStopResult, RuleMigrationTaskStartResult } from './task/types'; + +export interface StoredRuleMigration extends RuleMigration { + _id: string; + _index: string; +} export interface SiemRulesMigrationsSetupParams { esClusterClient: IClusterClient; @@ -16,15 +35,28 @@ export interface SiemRulesMigrationsSetupParams { tasksTimeoutMs?: number; } -export interface SiemRuleMigrationsGetClientParams { +export interface SiemRuleMigrationsCreateClientParams { request: KibanaRequest; + currentUser: AuthenticatedUser | null; spaceId: string; } -export interface RuleMigrationSearchParams { - migration_id?: string; +export interface SiemRuleMigrationsStartTaskParams { + migrationId: string; + connectorId: string; + invocationConfig: RunnableConfig; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; } + export interface SiemRuleMigrationsClient { - create: (body: RuleMigration[]) => Promise; - search: (params: RuleMigrationSearchParams) => Promise; + data: RuleMigrationsDataClient; + task: { + start: (params: SiemRuleMigrationsStartTaskParams) => Promise; + stop: (migrationId: string) => Promise; + getStats: (migrationId: string) => Promise; + getAllStats: () => Promise; + }; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts index 3d9e5b9fe179b..adf77756cce34 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts @@ -8,9 +8,15 @@ import { loggingSystemMock, elasticsearchServiceMock, httpServerMock, + securityServiceMock, } from '@kbn/core/server/mocks'; import { SiemMigrationsService } from './siem_migrations_service'; -import { MockSiemRuleMigrationsService, mockSetup, mockGetClient } from './rules/__mocks__/mocks'; +import { + MockSiemRuleMigrationsService, + mockSetup, + mockCreateClient, + mockStop, +} from './rules/__mocks__/mocks'; import type { ConfigType } from '../../config'; jest.mock('./rules/siem_rule_migrations_service'); @@ -25,6 +31,7 @@ describe('SiemMigrationsService', () => { let siemMigrationsService: SiemMigrationsService; const kibanaVersion = '8.16.0'; + const currentUser = securityServiceMock.createMockAuthenticatedUser(); const esClusterClient = elasticsearchServiceMock.createClusterClient(); const logger = loggingSystemMock.createLogger(); @@ -57,17 +64,22 @@ describe('SiemMigrationsService', () => { }); }); - describe('when createClient is called', () => { + describe('when createRulesClient is called', () => { it('should create rules client', async () => { - const request = httpServerMock.createKibanaRequest(); - siemMigrationsService.createClient({ spaceId: 'default', request }); - expect(mockGetClient).toHaveBeenCalledWith({ spaceId: 'default', request }); + const createRulesClientParams = { + spaceId: 'default', + request: httpServerMock.createKibanaRequest(), + currentUser, + }; + siemMigrationsService.createRulesClient(createRulesClientParams); + expect(mockCreateClient).toHaveBeenCalledWith(createRulesClientParams); }); }); describe('when stop is called', () => { it('should trigger the pluginStop subject', async () => { siemMigrationsService.stop(); + expect(mockStop).toHaveBeenCalled(); expect(mockReplaySubject$.next).toHaveBeenCalled(); expect(mockReplaySubject$.complete).toHaveBeenCalled(); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts index b84281eb13d9b..7a85dd625feec 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts @@ -9,11 +9,8 @@ import type { Logger } from '@kbn/core/server'; import { ReplaySubject, type Subject } from 'rxjs'; import type { ConfigType } from '../../config'; import { SiemRuleMigrationsService } from './rules/siem_rule_migrations_service'; -import type { - SiemMigrationsClient, - SiemMigrationsSetupParams, - SiemMigrationsGetClientParams, -} from './types'; +import type { SiemMigrationsSetupParams, SiemMigrationsCreateClientParams } from './types'; +import type { SiemRuleMigrationsClient } from './rules/types'; export class SiemMigrationsService { private pluginStop$: Subject; @@ -30,13 +27,12 @@ export class SiemMigrationsService { } } - createClient(params: SiemMigrationsGetClientParams): SiemMigrationsClient { - return { - rules: this.rules.getClient(params), - }; + createRulesClient(params: SiemMigrationsCreateClientParams): SiemRuleMigrationsClient { + return this.rules.createClient(params); } stop() { + this.rules.stop(); this.pluginStop$.next(); this.pluginStop$.complete(); } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts index b5647ff65e214..d2af1e2518722 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts @@ -6,15 +6,11 @@ */ import type { IClusterClient } from '@kbn/core/server'; -import type { SiemRuleMigrationsClient, SiemRuleMigrationsGetClientParams } from './rules/types'; +import type { SiemRuleMigrationsCreateClientParams } from './rules/types'; export interface SiemMigrationsSetupParams { esClusterClient: IClusterClient; tasksTimeoutMs?: number; } -export type SiemMigrationsGetClientParams = SiemRuleMigrationsGetClientParams; - -export interface SiemMigrationsClient { - rules: SiemRuleMigrationsClient; -} +export type SiemMigrationsCreateClientParams = SiemRuleMigrationsCreateClientParams; diff --git a/x-pack/plugins/security_solution/server/plugin_contract.ts b/x-pack/plugins/security_solution/server/plugin_contract.ts index c7ec67c1b07fc..c178f0654d9bd 100644 --- a/x-pack/plugins/security_solution/server/plugin_contract.ts +++ b/x-pack/plugins/security_solution/server/plugin_contract.ts @@ -45,6 +45,7 @@ import type { SharePluginStart } from '@kbn/share-plugin/server'; import type { GuidedOnboardingPluginSetup } from '@kbn/guided-onboarding-plugin/server'; import type { PluginSetup as UnifiedSearchServerPluginSetup } from '@kbn/unified-search-plugin/server'; import type { ElasticAssistantPluginStart } from '@kbn/elastic-assistant-plugin/server'; +import type { InferenceServerStart } from '@kbn/inference-plugin/server'; import type { ProductFeaturesService } from './lib/product_features_service/product_features_service'; import type { ExperimentalFeatures } from '../common'; @@ -88,6 +89,7 @@ export interface SecuritySolutionPluginStartDependencies { telemetry?: TelemetryPluginStart; share: SharePluginStart; actions: ActionsPluginStartContract; + inference: InferenceServerStart; } export interface SecuritySolutionPluginSetup { diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index f91f3c055a25b..fbe7be692e523 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -166,10 +166,16 @@ export class RequestContextFactory implements IRequestContextFactory { }) ), - getSiemMigrationsClient: memoize(() => - siemMigrationsService.createClient({ request, spaceId: getSpaceId() }) + getSiemRuleMigrationsClient: memoize(() => + siemMigrationsService.createRulesClient({ + request, + currentUser: coreContext.security.authc.getCurrentUser(), + spaceId: getSpaceId(), + }) ), + getInferenceClient: memoize(() => startPlugins.inference.getClient({ request })), + getExceptionListClient: () => { if (!lists) { return null; diff --git a/x-pack/plugins/security_solution/server/types.ts b/x-pack/plugins/security_solution/server/types.ts index 1355904dbe7f7..7afbf5dcff6d2 100644 --- a/x-pack/plugins/security_solution/server/types.ts +++ b/x-pack/plugins/security_solution/server/types.ts @@ -20,6 +20,7 @@ import type { AlertsClient, IRuleDataService } from '@kbn/rule-registry-plugin/s import type { Readable } from 'stream'; import type { AuditLogger } from '@kbn/security-plugin-types-server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; import type { Immutable } from '../common/endpoint/types'; import { AppClient } from './client'; import type { ConfigType } from './config'; @@ -35,7 +36,7 @@ import type { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk import type { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality'; import type { IDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface'; import type { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client'; -import type { SiemMigrationsClient } from './lib/siem_migrations/types'; +import type { SiemRuleMigrationsClient } from './lib/siem_migrations/rules/types'; export { AppClient }; export interface SecuritySolutionApiRequestHandlerContext { @@ -58,7 +59,8 @@ export interface SecuritySolutionApiRequestHandlerContext { getRiskScoreDataClient: () => RiskScoreDataClient; getAssetCriticalityDataClient: () => AssetCriticalityDataClient; getEntityStoreDataClient: () => EntityStoreDataClient; - getSiemMigrationsClient: () => SiemMigrationsClient; + getSiemRuleMigrationsClient: () => SiemRuleMigrationsClient; + getInferenceClient: () => InferenceClient; } export type SecuritySolutionRequestHandlerContext = CustomRequestHandlerContext<{ diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index cbdd2aed3496f..df743a666108e 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -229,5 +229,6 @@ "@kbn/data-stream-adapter", "@kbn/core-lifecycle-server", "@kbn/core-user-profile-common", + "@kbn/langchain", ] } diff --git a/x-pack/test/api_integration/services/security_solution_api.gen.ts b/x-pack/test/api_integration/services/security_solution_api.gen.ts index 1ddbbf2ed7365..0b1338fee46e2 100644 --- a/x-pack/test/api_integration/services/security_solution_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_api.gen.ts @@ -95,6 +95,8 @@ import { GetRuleExecutionResultsRequestQueryInput, GetRuleExecutionResultsRequestParamsInput, } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.gen'; +import { GetRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationStatsRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { GetTimelineRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timeline/get_timeline_route.gen'; import { GetTimelinesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timelines/get_timelines_route.gen'; import { ImportRulesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/import_rules/import_rules_route.gen'; @@ -127,7 +129,12 @@ import { SetAlertAssigneesRequestBodyInput } from '@kbn/security-solution-plugin import { SetAlertsStatusRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/signals/set_signal_status/set_signals_status_route.gen'; import { SetAlertTagsRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/alert_tags/set_alert_tags/set_alert_tags.gen'; import { StartEntityEngineRequestParamsInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/engine/start.gen'; +import { + StartRuleMigrationRequestParamsInput, + StartRuleMigrationRequestBodyInput, +} from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { StopEntityEngineRequestParamsInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/engine/stop.gen'; +import { StopRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { SuggestUserProfilesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/users/suggest_user_profiles_route.gen'; import { TriggerRiskScoreCalculationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/risk_engine/entity_calculation_route.gen'; import { UpdateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/update_rule/update_rule_route.gen'; @@ -755,6 +762,16 @@ finalize it. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Retrieves the rule migrations stats for all migrations stored in the system + */ + getAllStatsRuleMigration(kibanaSpace: string = 'default') { + return supertest + .get(routeWithNamespace('/internal/siem_migrations/rules/stats', kibanaSpace)) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, /** * Get the criticality record for a specific asset. */ @@ -909,11 +926,31 @@ finalize it. .query(props.query); }, /** - * Retrieves the rule migrations stored in the system + * Retrieves the rule documents stored in the system given the rule migration id */ - getRuleMigration(kibanaSpace: string = 'default') { + getRuleMigration(props: GetRuleMigrationProps, kibanaSpace: string = 'default') { return supertest - .get(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace)) + .get( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, + /** + * Retrieves the stats of a SIEM rules migration using the migration id provided + */ + getRuleMigrationStats(props: GetRuleMigrationStatsProps, kibanaSpace: string = 'default') { + return supertest + .get( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/stats', props.params), + kibanaSpace + ) + ) .set('kbn-xsrf', 'true') .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); @@ -1260,6 +1297,22 @@ detection engine rules. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Starts a SIEM rules migration using the migration id provided + */ + startRuleMigration(props: StartRuleMigrationProps, kibanaSpace: string = 'default') { + return supertest + .put( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/start', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send(props.body as object); + }, stopEntityEngine(props: StopEntityEngineProps, kibanaSpace: string = 'default') { return supertest .post( @@ -1272,6 +1325,21 @@ detection engine rules. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Stops a running SIEM rules migration using the migration id provided + */ + stopRuleMigration(props: StopRuleMigrationProps, kibanaSpace: string = 'default') { + return supertest + .put( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/stop', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, /** * Suggests user profiles. */ @@ -1490,6 +1558,12 @@ export interface GetRuleExecutionResultsProps { query: GetRuleExecutionResultsRequestQueryInput; params: GetRuleExecutionResultsRequestParamsInput; } +export interface GetRuleMigrationProps { + params: GetRuleMigrationRequestParamsInput; +} +export interface GetRuleMigrationStatsProps { + params: GetRuleMigrationStatsRequestParamsInput; +} export interface GetTimelineProps { query: GetTimelineRequestQueryInput; } @@ -1562,9 +1636,16 @@ export interface SetAlertTagsProps { export interface StartEntityEngineProps { params: StartEntityEngineRequestParamsInput; } +export interface StartRuleMigrationProps { + params: StartRuleMigrationRequestParamsInput; + body: StartRuleMigrationRequestBodyInput; +} export interface StopEntityEngineProps { params: StopEntityEngineRequestParamsInput; } +export interface StopRuleMigrationProps { + params: StopRuleMigrationRequestParamsInput; +} export interface SuggestUserProfilesProps { query: SuggestUserProfilesRequestQueryInput; }