From 4740371fabb6fcbdf658a401efc77aeb911be16c Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:03:19 +0000 Subject: [PATCH] [Security Solution][Detection Engine] adds legacy siem signals telemetry (#202671) ## Summary - partly addresses https://github.com/elastic/kibana/issues/195523 - adds snapshot telemetry that shows number of legacy siem signals and number of spaces they are in - while working on PR, discovered and fixed few issues in APIs - get migration status API did not work correctly with new `.alerts-*` indices, listing them as outdated - finalize migration API did account for spaces, when adding alias to migrated index - remove migration API failed due to lack of permissions to removed migration task from `.tasks` index ### How to test #### How to create legacy siem index? run script that used for FTR tests ```bash node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index ``` These would create legacy siem indices. But be aware, it might break Kibana .alerts indices creation. But sufficient for testing #### How to test snapshot telemetry Snapshot For snapshot telemetry use [API](https://docs.elastic.dev/telemetry/collection/snapshot-telemetry#telemetry-usage-payload-api) call OR Check snapshots in Kibana adv settings -> Global Settings Tab -> Usage collection section -> Click on cluster data example link -> Check `legacy_siem_signals ` fields in flyout
Snapshot telemetry Screenshot 2024-12-03 at 13 08 03
--------- Co-authored-by: Ryland Herrick (cherry picked from commit 8821e034e9c6cc4ad42915e54b429defd6b970b5) --- .../src/get_index_aliases/index.ts | 10 +- .../migrations/delete_migration.ts | 4 +- .../migrations/finalize_migration.test.ts | 3 + .../migrations/finalize_migration.ts | 3 + .../get_index_alias_per_space.test.ts | 88 +++++++++ .../migrations/get_index_alias_per_space.ts | 52 +++++ .../get_latest_index_template_version.test.ts | 59 ++++++ .../get_latest_index_template_version.ts | 34 ++++ .../get_non_migrated_signals_info.test.ts | 177 ++++++++++++++++++ .../get_non_migrated_signals_info.ts | 131 +++++++++++++ .../migrations/migration_service.ts | 4 +- .../migrations/replace_signals_index_alias.ts | 7 +- .../finalize_signals_migration_route.ts | 2 + .../get_signals_migration_status_route.ts | 6 +- .../security_solution/server/plugin.ts | 1 + .../server/usage/collector.ts | 17 ++ .../usage/detections/get_initial_usage.ts | 3 + .../usage/detections/get_metrics.test.ts | 6 + .../server/usage/detections/get_metrics.ts | 13 +- .../legacy_siem_signals/get_initial_usage.ts | 13 ++ .../get_legacy_siem_signals_metrics.ts | 33 ++++ .../detections/legacy_siem_signals/types.ts | 10 + .../server/usage/detections/types.ts | 3 + .../security_solution/server/usage/types.ts | 1 + .../schema/xpack_plugins.json | 16 ++ .../migrations/delete_alerts_migrations.ts | 7 +- .../alerts/migrations/delete_migrations.ts | 25 +++ 27 files changed, 714 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_initial_usage.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_legacy_siem_signals_metrics.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/types.ts diff --git a/packages/kbn-securitysolution-es-utils/src/get_index_aliases/index.ts b/packages/kbn-securitysolution-es-utils/src/get_index_aliases/index.ts index 99c6ffc3d05b7..29e526350be6c 100644 --- a/packages/kbn-securitysolution-es-utils/src/get_index_aliases/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/get_index_aliases/index.ts @@ -20,26 +20,30 @@ interface IndexAlias { * * @param esClient An {@link ElasticsearchClient} * @param alias alias name used to filter results + * @param index index name used to filter results * * @returns an array of {@link IndexAlias} objects */ export const getIndexAliases = async ({ esClient, alias, + index, }: { esClient: ElasticsearchClient; alias: string; + index?: string; }): Promise => { const response = await esClient.indices.getAlias( { name: alias, + ...(index ? { index } : {}), }, { meta: true } ); - return Object.keys(response.body).map((index) => ({ + return Object.keys(response.body).map((indexName) => ({ alias, - index, - isWriteIndex: response.body[index].aliases[alias]?.is_write_index === true, + index: indexName, + isWriteIndex: response.body[indexName].aliases[alias]?.is_write_index === true, })); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/delete_migration.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/delete_migration.ts index 45098f8dea239..04f15570434a3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/delete_migration.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/delete_migration.ts @@ -14,7 +14,6 @@ import type { SignalsMigrationSO } from './saved_objects_schema'; /** * Deletes a completed migration: * * deletes the migration SO - * * deletes the underlying task document * * applies deletion policy to the relevant index * * @param esClient An {@link ElasticsearchClient} @@ -40,7 +39,7 @@ export const deleteMigration = async ({ return migration; } - const { destinationIndex, sourceIndex, taskId } = migration.attributes; + const { destinationIndex, sourceIndex } = migration.attributes; if (isMigrationFailed(migration)) { await applyMigrationCleanupPolicy({ @@ -57,7 +56,6 @@ export const deleteMigration = async ({ }); } - await esClient.delete({ index: '.tasks', id: taskId }); await deleteMigrationSavedObject({ id: migration.id, soClient }); return migration; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.test.ts index 6c855a5b77748..51f81ad75e405 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.test.ts @@ -40,6 +40,7 @@ describe('finalizeMigration', () => { signalsAlias: 'my-signals-alias', soClient, username: 'username', + legacySiemSignalsAlias: '.siem-signals-default', }); expect(updateMigrationSavedObject).not.toHaveBeenCalled(); @@ -54,6 +55,7 @@ describe('finalizeMigration', () => { signalsAlias: 'my-signals-alias', soClient, username: 'username', + legacySiemSignalsAlias: '.siem-signals-default', }); expect(updateMigrationSavedObject).not.toHaveBeenCalled(); @@ -72,6 +74,7 @@ describe('finalizeMigration', () => { signalsAlias: 'my-signals-alias', soClient, username: 'username', + legacySiemSignalsAlias: '.siem-signals-default', }); expect(updateMigrationSavedObject).toHaveBeenCalledWith( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.ts index e9ce2a4a641a6..3aca53c1422d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.ts @@ -35,12 +35,14 @@ export const finalizeMigration = async ({ signalsAlias, soClient, username, + legacySiemSignalsAlias, }: { esClient: ElasticsearchClient; migration: SignalsMigrationSO; signalsAlias: string; soClient: SavedObjectsClientContract; username: string; + legacySiemSignalsAlias: string; }): Promise => { if (!isMigrationPending(migration)) { return migration; @@ -86,6 +88,7 @@ export const finalizeMigration = async ({ esClient, newIndex: destinationIndex, oldIndex: sourceIndex, + legacySiemSignalsAlias, }); const updatedMigration = await updateMigrationSavedObject({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.test.ts new file mode 100644 index 0000000000000..a2361c6b7aeee --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { getIndexAliasPerSpace } from './get_index_alias_per_space'; + +describe('getIndexAliasPerSpace', () => { + let esClient: ReturnType; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('returns object with index alias and space', async () => { + esClient.indices.getAlias.mockResponseOnce({ + '.siem-signals-default-old-one': { + aliases: { + '.siem-signals-default': { + is_write_index: false, + }, + }, + }, + '.siem-signals-another-1-legacy': { + aliases: { + '.siem-signals-another-1': { + is_write_index: false, + }, + }, + }, + }); + + const result = await getIndexAliasPerSpace({ + esClient, + signalsIndex: '.siem-signals', + signalsAliasAllSpaces: '.siem-signals-*', + }); + + expect(result).toEqual({ + '.siem-signals-another-1-legacy': { + alias: '.siem-signals-another-1', + indexName: '.siem-signals-another-1-legacy', + space: 'another-1', + }, + '.siem-signals-default-old-one': { + alias: '.siem-signals-default', + indexName: '.siem-signals-default-old-one', + space: 'default', + }, + }); + }); + + it('filters out .internal.alert indices', async () => { + esClient.indices.getAlias.mockResponseOnce({ + '.siem-signals-default-old-one': { + aliases: { + '.siem-signals-default': { + is_write_index: false, + }, + }, + }, + '.internal.alerts-security.alerts-another-2-000001': { + aliases: { + '.siem-signals-another-2': { + is_write_index: false, + }, + }, + }, + }); + + const result = await getIndexAliasPerSpace({ + esClient, + signalsIndex: '.siem-signals', + signalsAliasAllSpaces: '.siem-signals-*', + }); + + expect(result).toEqual({ + '.siem-signals-default-old-one': { + alias: '.siem-signals-default', + indexName: '.siem-signals-default-old-one', + space: 'default', + }, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.ts new file mode 100644 index 0000000000000..04895f0d74f39 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { ElasticsearchClient } from '@kbn/core/server'; + +interface IndexAlias { + alias: string; + space: string; + indexName: string; +} + +/** + * Retrieves index, its alias and Kibana space + */ +export const getIndexAliasPerSpace = async ({ + esClient, + signalsIndex, + signalsAliasAllSpaces, +}: { + esClient: ElasticsearchClient; + signalsIndex: string; + signalsAliasAllSpaces: string; +}): Promise> => { + const response = await esClient.indices.getAlias( + { + name: signalsAliasAllSpaces, + }, + { meta: true } + ); + + const indexAliasesMap = Object.keys(response.body).reduce>( + (acc, indexName) => { + if (!indexName.startsWith('.internal.alerts-')) { + const alias = Object.keys(response.body[indexName].aliases)[0]; + + acc[indexName] = { + alias, + space: alias.replace(`${signalsIndex}-`, ''), + indexName, + }; + } + + return acc; + }, + {} + ); + + return indexAliasesMap; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.test.ts new file mode 100644 index 0000000000000..ea48c51aabcc5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { IndicesGetIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { getLatestIndexTemplateVersion } from './get_latest_index_template_version'; + +describe('getIndexAliasPerSpace', () => { + let esClient: ReturnType; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('returns latest index template version', async () => { + esClient.indices.getIndexTemplate.mockResponseOnce({ + index_templates: [ + { index_template: { version: 77 } }, + { index_template: { version: 10 } }, + { index_template: { version: 23 } }, + { index_template: { version: 0 } }, + ], + } as IndicesGetIndexTemplateResponse); + + const version = await getLatestIndexTemplateVersion({ + esClient, + name: '.siem-signals-*', + }); + + expect(version).toBe(77); + }); + + it('returns 0 if templates empty', async () => { + esClient.indices.getIndexTemplate.mockResponseOnce({ + index_templates: [], + }); + + const version = await getLatestIndexTemplateVersion({ + esClient, + name: '.siem-signals-*', + }); + + expect(version).toBe(0); + }); + + it('returns 0 if request fails', async () => { + esClient.indices.getIndexTemplate.mockRejectedValueOnce('Failure'); + + const version = await getLatestIndexTemplateVersion({ + esClient, + name: '.siem-signals-*', + }); + + expect(version).toBe(0); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.ts new file mode 100644 index 0000000000000..b06a14adc2ce2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { ElasticsearchClient } from '@kbn/core/server'; + +/** + * Retrieves the latest version of index template + * There are can be multiple index templates across different Kibana spaces, + * so we get them all and return the latest(greatest) number + */ +export const getLatestIndexTemplateVersion = async ({ + esClient, + name, +}: { + esClient: ElasticsearchClient; + name: string; +}): Promise => { + let latestTemplateVersion: number; + try { + const response = await esClient.indices.getIndexTemplate({ name }); + const versions = response.index_templates.map( + (template) => template.index_template.version ?? 0 + ); + + latestTemplateVersion = versions.length ? Math.max(...versions) : 0; + } catch (e) { + latestTemplateVersion = 0; + } + + return latestTemplateVersion; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts new file mode 100644 index 0000000000000..36252ab792342 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +import { getNonMigratedSignalsInfo } from './get_non_migrated_signals_info'; +import { getIndexVersionsByIndex } from './get_index_versions_by_index'; +import { getSignalVersionsByIndex } from './get_signal_versions_by_index'; +import { getLatestIndexTemplateVersion } from './get_latest_index_template_version'; +import { getIndexAliasPerSpace } from './get_index_alias_per_space'; + +jest.mock('./get_index_versions_by_index', () => ({ getIndexVersionsByIndex: jest.fn() })); +jest.mock('./get_signal_versions_by_index', () => ({ getSignalVersionsByIndex: jest.fn() })); +jest.mock('./get_latest_index_template_version', () => ({ + getLatestIndexTemplateVersion: jest.fn(), +})); +jest.mock('./get_index_alias_per_space', () => ({ getIndexAliasPerSpace: jest.fn() })); + +const getIndexVersionsByIndexMock = getIndexVersionsByIndex as jest.Mock; +const getSignalVersionsByIndexMock = getSignalVersionsByIndex as jest.Mock; +const getLatestIndexTemplateVersionMock = getLatestIndexTemplateVersion as jest.Mock; +const getIndexAliasPerSpaceMock = getIndexAliasPerSpace as jest.Mock; + +const TEMPLATE_VERSION = 77; + +describe('getNonMigratedSignalsInfo', () => { + let esClient: ReturnType; + const logger = loggerMock.create(); + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + + getLatestIndexTemplateVersionMock.mockReturnValue(TEMPLATE_VERSION); + getIndexVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': 10, + '.siem-signals-default-old-one': 42, + }); + getSignalVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': [{ count: 2, version: 10 }], + }); + getIndexAliasPerSpaceMock.mockReturnValue({ + '.siem-signals-another-1-legacy': { + alias: '.siem-signals-another-1', + indexName: '.siem-signals-another-1-legacy', + space: 'another-1', + }, + '.siem-signals-default-old-one': { + alias: '.siem-signals-default', + indexName: '.siem-signals-default-old-one', + space: 'default', + }, + }); + }); + + it('returns empty results if no siem indices found', async () => { + getIndexAliasPerSpaceMock.mockReturnValue({}); + + const result = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: 'siem-signals', + logger, + }); + + expect(result).toEqual({ + isMigrationRequired: false, + spaces: [], + indices: [], + }); + }); + + it('returns empty when error happens', async () => { + getLatestIndexTemplateVersionMock.mockRejectedValueOnce(new Error('Test failure')); + const debugSpy = jest.spyOn(logger, 'debug'); + + const result = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: 'siem-signals', + logger, + }); + + expect(result).toEqual({ + isMigrationRequired: false, + spaces: [], + indices: [], + }); + expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('Test failure')); + }); + + it('returns empty results if no siem indices or signals outdated', async () => { + getIndexVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': TEMPLATE_VERSION, + '.siem-signals-default-old-one': TEMPLATE_VERSION, + }); + getSignalVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': [{ count: 2, version: TEMPLATE_VERSION }], + }); + + const result = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: 'siem-signals', + logger, + }); + + expect(result).toEqual({ + isMigrationRequired: false, + spaces: [], + indices: [], + }); + }); + it('returns results for outdated index', async () => { + getIndexVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': TEMPLATE_VERSION, + '.siem-signals-default-old-one': 16, + }); + getSignalVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': [{ count: 2, version: TEMPLATE_VERSION }], + }); + + const result = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: 'siem-signals', + logger, + }); + + expect(result).toEqual({ + indices: ['.siem-signals-default-old-one'], + isMigrationRequired: true, + spaces: ['default'], + }); + }); + it('returns results for outdated signals in index', async () => { + getIndexVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': TEMPLATE_VERSION, + '.siem-signals-default-old-one': TEMPLATE_VERSION, + }); + getSignalVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': [{ count: 2, version: 12 }], + }); + + const result = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: 'siem-signals', + logger, + }); + + expect(result).toEqual({ + indices: ['.siem-signals-another-1-legacy'], + isMigrationRequired: true, + spaces: ['another-1'], + }); + }); + it('returns indices in multiple spaces', async () => { + getIndexVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': 11, + '.siem-signals-default-old-one': 11, + }); + getSignalVersionsByIndexMock.mockReturnValue({ + '.siem-signals-another-1-legacy': [{ count: 2, version: 11 }], + }); + + const result = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: 'siem-signals', + logger, + }); + + expect(result).toEqual({ + indices: ['.siem-signals-another-1-legacy', '.siem-signals-default-old-one'], + isMigrationRequired: true, + spaces: ['another-1', 'default'], + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts new file mode 100644 index 0000000000000..d1f561fb3846c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import type { IndexVersionsByIndex } from './get_index_versions_by_index'; +import { getIndexVersionsByIndex } from './get_index_versions_by_index'; +import { + getSignalVersionsByIndex, + type SignalVersionsByIndex, +} from './get_signal_versions_by_index'; +import { isOutdated as getIsOutdated, signalsAreOutdated } from './helpers'; +import { getLatestIndexTemplateVersion } from './get_latest_index_template_version'; +import { getIndexAliasPerSpace } from './get_index_alias_per_space'; + +interface OutdatedSpaces { + isMigrationRequired: boolean; + spaces: string[]; + indices: string[]; +} + +/** + * gets lists of spaces and non-migrated signal indices + */ +export const getNonMigratedSignalsInfo = async ({ + esClient, + signalsIndex, + logger, +}: { + esClient: ElasticsearchClient; + signalsIndex: string; + logger: Logger; +}): Promise => { + const signalsAliasAllSpaces = `${signalsIndex}-*`; + + try { + const latestTemplateVersion = await getLatestIndexTemplateVersion({ + esClient, + name: signalsAliasAllSpaces, + }); + const indexAliasesMap = await getIndexAliasPerSpace({ + esClient, + signalsAliasAllSpaces, + signalsIndex, + }); + + const indices = Object.keys(indexAliasesMap); + + if (indices.length === 0) { + return { + isMigrationRequired: false, + spaces: [], + indices: [], + }; + } + + let indexVersionsByIndex: IndexVersionsByIndex = {}; + try { + indexVersionsByIndex = await getIndexVersionsByIndex({ + esClient, + index: indices, + }); + } catch (e) { + logger.debug( + `Getting information about legacy siem signals index version failed:"${e?.message}"` + ); + } + + let signalVersionsByIndex: SignalVersionsByIndex = {}; + try { + signalVersionsByIndex = await getSignalVersionsByIndex({ + esClient, + index: indices, + }); + } catch (e) { + logger.debug(`Getting information about legacy siem signals versions failed:"${e?.message}"`); + } + + const outdatedIndices = indices.reduce>( + (acc, indexName) => { + const version = indexVersionsByIndex[indexName] ?? 0; + const signalVersions = signalVersionsByIndex[indexName] ?? []; + + const isOutdated = + getIsOutdated({ current: version, target: latestTemplateVersion }) || + signalsAreOutdated({ signalVersions, target: latestTemplateVersion }); + + if (isOutdated) { + acc.push({ + indexName, + space: indexAliasesMap[indexName].space, + }); + } + + return acc; + }, + [] + ); + + const outdatedIndexNames = outdatedIndices.map((outdatedIndex) => outdatedIndex.indexName); + + // remove duplicated spaces + const spaces = [...new Set(outdatedIndices.map((indexStatus) => indexStatus.space))]; + const isMigrationRequired = outdatedIndices.length > 0; + + logger.debug( + isMigrationRequired + ? `Legacy siem signals indices require migration: "${outdatedIndexNames.join( + ', ' + )}" in "${spaces.join(', ')}" spaces` + : 'No legacy siem indices require migration' + ); + + return { + isMigrationRequired, + spaces, + indices: outdatedIndexNames, + }; + } catch (e) { + logger.debug(`Getting information about legacy siem signals failed:"${e?.message}"`); + return { + isMigrationRequired: false, + spaces: [], + indices: [], + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_service.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_service.ts index 5a4399bd6389c..5530f0a80c5d2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_service.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_service.ts @@ -22,6 +22,7 @@ export interface CreateParams { export interface FinalizeParams { signalsAlias: string; migration: SignalsMigrationSO; + legacySiemSignalsAlias: string; } export interface DeleteParams { @@ -59,13 +60,14 @@ export const signalsMigrationService = ({ username, }); }, - finalize: ({ migration, signalsAlias }) => + finalize: ({ migration, signalsAlias, legacySiemSignalsAlias }) => finalizeMigration({ esClient, migration, signalsAlias, soClient, username, + legacySiemSignalsAlias, }), delete: ({ migration, signalsAlias }) => deleteMigration({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/replace_signals_index_alias.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/replace_signals_index_alias.ts index ad77e64a55ac7..984707959005c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/replace_signals_index_alias.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/replace_signals_index_alias.ts @@ -26,11 +26,13 @@ export const replaceSignalsIndexAlias = async ({ esClient, newIndex, oldIndex, + legacySiemSignalsAlias, }: { alias: string; esClient: ElasticsearchClient; newIndex: string; oldIndex: string; + legacySiemSignalsAlias: string; }): Promise => { await esClient.indices.updateAliases({ body: { @@ -40,12 +42,11 @@ export const replaceSignalsIndexAlias = async ({ ], }, }); - // TODO: space-aware? await esClient.indices.updateAliases({ body: { actions: [ - { remove: { index: oldIndex, alias: '.siem-signals-default' } }, - { add: { index: newIndex, alias: '.siem-signals-default', is_write_index: false } }, + { remove: { index: oldIndex, alias: legacySiemSignalsAlias } }, + { add: { index: newIndex, alias: legacySiemSignalsAlias, is_write_index: false } }, ], }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.ts index 0ff0220056e73..4421a116def76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.ts @@ -74,6 +74,7 @@ export const finalizeSignalsMigrationRoute = ( }); const spaceId = securitySolution.getSpaceId(); + const legacySiemSignalsAlias = appClient.getSignalsIndex(); const signalsAlias = ruleDataService.getResourceName(`security.alerts-${spaceId}`); const finalizeResults = await Promise.all( migrations.map(async (migration) => { @@ -81,6 +82,7 @@ export const finalizeSignalsMigrationRoute = ( const finalizedMigration = await migrationService.finalize({ migration, signalsAlias, + legacySiemSignalsAlias, }); if (isMigrationFailed(finalizedMigration)) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/get_signals_migration_status_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/get_signals_migration_status_route.ts index 15f64c7f96c41..8bee9b1947c2f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/get_signals_migration_status_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/get_signals_migration_status_route.ts @@ -65,7 +65,11 @@ export const getSignalsMigrationStatusRoute = ( const signalsAlias = appClient.getSignalsIndex(); const currentVersion = await getTemplateVersion({ alias: signalsAlias, esClient }); - const indexAliases = await getIndexAliases({ alias: signalsAlias, esClient }); + const indexAliases = await getIndexAliases({ + alias: signalsAlias, + esClient, + index: `${signalsAlias}-*`, + }); const signalsIndices = indexAliases.map((indexAlias) => indexAlias.index); const indicesInRange = await getSignalsIndicesInRange({ esClient, diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2c7a62a3a7411..2029ca4df28c6 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -280,6 +280,7 @@ export class Plugin implements ISecuritySolutionPlugin { all: allRiskScoreIndexPattern, latest: latestRiskScoreIndexPattern, }, + legacySignalsIndex: config.signalsIndex, }); this.telemetryUsageCounter = plugins.usageCollection?.createUsageCounter(APP_ID); diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index ca016d07d5099..aa507cd683db5 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -32,6 +32,7 @@ export const registerCollector: RegisterCollector = ({ usageCollection, logger, riskEngineIndexPatterns, + legacySignalsIndex, }) => { if (!usageCollection) { logger.debug('Usage collection is undefined, therefore returning early without registering it'); @@ -3076,6 +3077,21 @@ export const registerCollector: RegisterCollector = ({ }, }, }, + legacy_siem_signals: { + non_migrated_indices_total: { + type: 'long', + _meta: { + description: 'Total number of non migrated legacy siem signals indices', + }, + }, + spaces_total: { + type: 'long', + _meta: { + description: + 'Total number of Kibana spaces that have non migrated legacy siem signals indices', + }, + }, + }, }, endpointMetrics: { unique_endpoint_count: { @@ -3130,6 +3146,7 @@ export const registerCollector: RegisterCollector = ({ savedObjectsClient, logger, mlClient: ml, + legacySignalsIndex, }), getEndpointMetrics({ esClient, logger }), getDashboardMetrics({ diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts index 6252b865c0ec9..538ea2509c463 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts @@ -9,6 +9,8 @@ import type { DetectionMetrics } from './types'; import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage'; import { getInitialEventLogUsage, getInitialRulesUsage } from './rules/get_initial_usage'; +// eslint-disable-next-line no-restricted-imports +import { getInitialLegacySiemSignalsUsage } from './legacy_siem_signals/get_initial_usage'; /** * Initial detection metrics initialized. @@ -23,4 +25,5 @@ export const getInitialDetectionMetrics = (): DetectionMetrics => ({ detection_rule_usage: getInitialRulesUsage(), detection_rule_status: getInitialEventLogUsage(), }, + legacy_siem_signals: getInitialLegacySiemSignalsUsage(), }); diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts index be5044fbb4e21..cb5006799a1cf 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts @@ -57,6 +57,7 @@ describe('Detections Usage and Metrics', () => { savedObjectsClient, logger, mlClient, + legacySignalsIndex: '', }); expect(result).toEqual(getInitialDetectionMetrics()); }); @@ -79,6 +80,7 @@ describe('Detections Usage and Metrics', () => { savedObjectsClient, logger, mlClient, + legacySignalsIndex: '', }); expect(result).toEqual({ @@ -154,6 +156,7 @@ describe('Detections Usage and Metrics', () => { savedObjectsClient, logger, mlClient, + legacySignalsIndex: '', }); expect(result).toEqual({ @@ -210,6 +213,7 @@ describe('Detections Usage and Metrics', () => { savedObjectsClient, logger, mlClient, + legacySignalsIndex: '', }); expect(result).toEqual({ @@ -290,6 +294,7 @@ describe('Detections Usage and Metrics', () => { savedObjectsClient, logger, mlClient, + legacySignalsIndex: '', }); expect(result).toEqual(getInitialDetectionMetrics()); }); @@ -329,6 +334,7 @@ describe('Detections Usage and Metrics', () => { savedObjectsClient, logger, mlClient, + legacySignalsIndex: '', }); expect(result).toEqual( diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts index 904c80debf5da..61badd153c65f 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts @@ -13,6 +13,10 @@ import { getMlJobMetrics } from './ml_jobs/get_metrics'; import { getRuleMetrics } from './rules/get_metrics'; import { getInitialEventLogUsage, getInitialRulesUsage } from './rules/get_initial_usage'; import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage'; +// eslint-disable-next-line no-restricted-imports +import { getInitialLegacySiemSignalsUsage } from './legacy_siem_signals/get_initial_usage'; +// eslint-disable-next-line no-restricted-imports +import { getLegacySiemSignalsUsage } from './legacy_siem_signals/get_legacy_siem_signals_metrics'; export interface GetDetectionsMetricsOptions { signalsIndex: string; @@ -21,6 +25,7 @@ export interface GetDetectionsMetricsOptions { logger: Logger; mlClient: MlPluginSetup | undefined; eventLogIndex: string; + legacySignalsIndex: string; } export const getDetectionsMetrics = async ({ @@ -30,10 +35,12 @@ export const getDetectionsMetrics = async ({ savedObjectsClient, logger, mlClient, + legacySignalsIndex, }: GetDetectionsMetricsOptions): Promise => { - const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([ + const [mlJobMetrics, detectionRuleMetrics, legacySiemSignalsUsage] = await Promise.allSettled([ getMlJobMetrics({ mlClient, savedObjectsClient, logger }), getRuleMetrics({ signalsIndex, eventLogIndex, esClient, savedObjectsClient, logger }), + getLegacySiemSignalsUsage({ signalsIndex: legacySignalsIndex, esClient, logger }), ]); return { @@ -49,5 +56,9 @@ export const getDetectionsMetrics = async ({ detection_rule_usage: getInitialRulesUsage(), detection_rule_status: getInitialEventLogUsage(), }, + legacy_siem_signals: + legacySiemSignalsUsage.status === 'fulfilled' + ? legacySiemSignalsUsage.value + : getInitialLegacySiemSignalsUsage(), }; }; diff --git a/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_initial_usage.ts new file mode 100644 index 0000000000000..df222f9750490 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_initial_usage.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { LegacySiemSignals } from './types'; + +export const getInitialLegacySiemSignalsUsage = (): LegacySiemSignals => ({ + non_migrated_indices_total: 0, + spaces_total: 0, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_legacy_siem_signals_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_legacy_siem_signals_metrics.ts new file mode 100644 index 0000000000000..a6d2b8bf06aff --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/get_legacy_siem_signals_metrics.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { getNonMigratedSignalsInfo } from '../../../lib/detection_engine/migrations/get_non_migrated_signals_info'; +import type { LegacySiemSignals } from './types'; + +export interface GetLegacySiemSignalsUsageOptions { + signalsIndex: string; + esClient: ElasticsearchClient; + logger: Logger; +} + +export const getLegacySiemSignalsUsage = async ({ + signalsIndex, + esClient, + logger, +}: GetLegacySiemSignalsUsageOptions): Promise => { + const { indices, spaces } = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex, + logger, + }); + + return { + non_migrated_indices_total: indices.length, + spaces_total: spaces.length, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/types.ts b/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/types.ts new file mode 100644 index 0000000000000..b4351b2e7808f --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/legacy_siem_signals/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export interface LegacySiemSignals { + non_migrated_indices_total: number; + spaces_total: number; +} diff --git a/x-pack/plugins/security_solution/server/usage/detections/types.ts b/x-pack/plugins/security_solution/server/usage/detections/types.ts index 2895e5c6f8b9a..3edbd028b6321 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/types.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/types.ts @@ -7,8 +7,11 @@ import type { MlJobUsageMetric } from './ml_jobs/types'; import type { RuleAdoption } from './rules/types'; +// eslint-disable-next-line no-restricted-imports +import type { LegacySiemSignals } from './legacy_siem_signals/types'; export interface DetectionMetrics { ml_jobs: MlJobUsageMetric; detection_rules: RuleAdoption; + legacy_siem_signals: LegacySiemSignals; } diff --git a/x-pack/plugins/security_solution/server/usage/types.ts b/x-pack/plugins/security_solution/server/usage/types.ts index 1df8f2d1388a0..fdaaac663ad41 100644 --- a/x-pack/plugins/security_solution/server/usage/types.ts +++ b/x-pack/plugins/security_solution/server/usage/types.ts @@ -33,6 +33,7 @@ export type CollectorDependencies = { all: string; latest: string; }; + legacySignalsIndex: string; } & Pick; export interface AlertBucket { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index eb9150f8482e2..34f755702c7b1 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -19470,6 +19470,22 @@ } } } + }, + "legacy_siem_signals": { + "properties": { + "non_migrated_indices_total": { + "type": "long", + "_meta": { + "description": "Total number of non migrated legacy siem signals indices" + } + }, + "spaces_total": { + "type": "long", + "_meta": { + "description": "Total number of Kibana spaces that have non migrated legacy siem signals indices" + } + } + } } } }, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/delete_alerts_migrations.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/delete_alerts_migrations.ts index 85911fc8ef7de..13c8ffc3c8ad0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/delete_alerts_migrations.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/delete_alerts_migrations.ts @@ -13,7 +13,7 @@ import { DETECTION_ENGINE_SIGNALS_MIGRATION_URL, } from '@kbn/security-solution-plugin/common/constants'; import { ROLES } from '@kbn/security-solution-plugin/common/test'; -import { deleteMigrations, getIndexNameFromLoad } from '../../../../../utils'; +import { deleteMigrationsIfExistent, getIndexNameFromLoad } from '../../../../../utils'; import { createAlertsIndex, deleteAllAlerts, @@ -84,10 +84,12 @@ export default ({ getService }: FtrProviderContext): void => { afterEach(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/signals/outdated_signals_index'); - await deleteMigrations({ + await deleteMigrationsIfExistent({ kbnClient, ids: [createdMigration.migration_id], }); + // we need to delete migrated index, otherwise create migration call(in beforeEach hook) will fail + await es.indices.delete({ index: createdMigration.migration_index }); await deleteAllAlerts(supertest, log, es); }); @@ -99,6 +101,7 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); const deletedMigration = body.migrations[0]; + expect(deletedMigration.error).to.eql(undefined); expect(deletedMigration.id).to.eql(createdMigration.migration_id); expect(deletedMigration.sourceIndex).to.eql(outdatedAlertsIndexName); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/migrations/delete_migrations.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/migrations/delete_migrations.ts index 9da2e76aba4f9..da230c862889c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/migrations/delete_migrations.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/migrations/delete_migrations.ts @@ -25,3 +25,28 @@ export const deleteMigrations = async ({ ) ); }; + +export const deleteMigrationsIfExistent = async ({ + ids, + kbnClient, +}: { + ids: string[]; + kbnClient: KbnClient; +}): Promise => { + await Promise.all( + ids.map(async (id) => { + try { + const res = await kbnClient.savedObjects.delete({ + id, + type: signalsMigrationType, + }); + return res; + } catch (e) { + // do not throw error when migration already deleted/not found + if (e?.response?.status !== 404) { + throw e; + } + } + }) + ); +};