[Security Solution][Detection Engine] adds legacy siem signals teleme…

…try (elastic#202671)

## Summary

- partly addresses elastic#195523
- adds snapshot telemetry that shows number of legacy siem signals and
number of spaces they are in
- while working on PR, discovered and fixed few issues in APIs
- get migration status API did not work correctly with new `.alerts-*`
indices, listing them as outdated
- finalize migration API did account for spaces, when adding alias to
migrated index
- remove migration API failed due to lack of permissions to removed
migration task from `.tasks` index

### How to test

#### How to create legacy siem index?

run script that used for FTR tests

```bash
node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index

```
These would create legacy siem indices. But be aware, it might break
Kibana .alerts indices creation. But sufficient for testing


#### How to test snapshot telemetry

Snapshot
For snapshot telemetry use
[API](https://docs.elastic.dev/telemetry/collection/snapshot-telemetry#telemetry-usage-payload-api)
call
OR
Check snapshots in Kibana adv settings -> Global Settings Tab -> Usage
collection section -> Click on cluster data example link -> Check
`legacy_siem_signals ` fields in flyout

<details>
<summary> Snapshot telemetry </summary>


<img width="2549" alt="Screenshot 2024-12-03 at 13 08 03"
src="https://github.com/user-attachments/assets/28ffe983-01c7-4435-a82a-9a968d32d5e0">


 </details>

---------

Co-authored-by: Ryland Herrick <[email protected]>

packages/kbn-securitysolution-es-utils/src/get_index_aliases/index.ts

@@ -20,26 +20,30 @@ interface IndexAlias {
  *
  * @param esClient An {@link ElasticsearchClient}
  * @param alias alias name used to filter results
+ * @param index index name used to filter results
  *
  * @returns an array of {@link IndexAlias} objects
  */
 export const getIndexAliases = async ({
   esClient,
   alias,
+  index,
 }: {
   esClient: ElasticsearchClient;
   alias: string;
+  index?: string;
 }): Promise<IndexAlias[]> => {
   const response = await esClient.indices.getAlias(
     {
       name: alias,
+      ...(index ? { index } : {}),
     },
     { meta: true }
   );
 
-  return Object.keys(response.body).map((index) => ({
+  return Object.keys(response.body).map((indexName) => ({
     alias,
-    index,
-    isWriteIndex: response.body[index].aliases[alias]?.is_write_index === true,
+    index: indexName,
+    isWriteIndex: response.body[indexName].aliases[alias]?.is_write_index === true,
   }));
 };

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/delete_migration.ts

@@ -14,7 +14,6 @@ import type { SignalsMigrationSO } from './saved_objects_schema';
 /**
  * Deletes a completed migration:
  *   * deletes the migration SO
- *   * deletes the underlying task document
  *   * applies deletion policy to the relevant index
  *
  * @param esClient An {@link ElasticsearchClient}
@@ -40,7 +39,7 @@ export const deleteMigration = async ({
     return migration;
   }
 
-  const { destinationIndex, sourceIndex, taskId } = migration.attributes;
+  const { destinationIndex, sourceIndex } = migration.attributes;
 
   if (isMigrationFailed(migration)) {
     await applyMigrationCleanupPolicy({
@@ -57,7 +56,6 @@ export const deleteMigration = async ({
     });
   }
 
-  await esClient.delete({ index: '.tasks', id: taskId });
   await deleteMigrationSavedObject({ id: migration.id, soClient });
 
   return migration;

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.test.ts

@@ -40,6 +40,7 @@ describe('finalizeMigration', () => {
       signalsAlias: 'my-signals-alias',
       soClient,
       username: 'username',
+      legacySiemSignalsAlias: '.siem-signals-default',
     });
 
     expect(updateMigrationSavedObject).not.toHaveBeenCalled();
@@ -54,6 +55,7 @@ describe('finalizeMigration', () => {
       signalsAlias: 'my-signals-alias',
       soClient,
       username: 'username',
+      legacySiemSignalsAlias: '.siem-signals-default',
     });
 
     expect(updateMigrationSavedObject).not.toHaveBeenCalled();
@@ -72,6 +74,7 @@ describe('finalizeMigration', () => {
       signalsAlias: 'my-signals-alias',
       soClient,
       username: 'username',
+      legacySiemSignalsAlias: '.siem-signals-default',
     });
 
     expect(updateMigrationSavedObject).toHaveBeenCalledWith(

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/finalize_migration.ts

@@ -35,12 +35,14 @@ export const finalizeMigration = async ({
   signalsAlias,
   soClient,
   username,
+  legacySiemSignalsAlias,
 }: {
   esClient: ElasticsearchClient;
   migration: SignalsMigrationSO;
   signalsAlias: string;
   soClient: SavedObjectsClientContract;
   username: string;
+  legacySiemSignalsAlias: string;
 }): Promise<SignalsMigrationSO> => {
   if (!isMigrationPending(migration)) {
     return migration;
@@ -86,6 +88,7 @@ export const finalizeMigration = async ({
     esClient,
     newIndex: destinationIndex,
     oldIndex: sourceIndex,
+    legacySiemSignalsAlias,
   });
 
   const updatedMigration = await updateMigrationSavedObject({

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.test.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
+import { getIndexAliasPerSpace } from './get_index_alias_per_space';
+
+describe('getIndexAliasPerSpace', () => {
+  let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
+
+  beforeEach(() => {
+    esClient = elasticsearchServiceMock.createElasticsearchClient();
+  });
+
+  it('returns object with index alias and space', async () => {
+    esClient.indices.getAlias.mockResponseOnce({
+      '.siem-signals-default-old-one': {
+        aliases: {
+          '.siem-signals-default': {
+            is_write_index: false,
+          },
+        },
+      },
+      '.siem-signals-another-1-legacy': {
+        aliases: {
+          '.siem-signals-another-1': {
+            is_write_index: false,
+          },
+        },
+      },
+    });
+
+    const result = await getIndexAliasPerSpace({
+      esClient,
+      signalsIndex: '.siem-signals',
+      signalsAliasAllSpaces: '.siem-signals-*',
+    });
+
+    expect(result).toEqual({
+      '.siem-signals-another-1-legacy': {
+        alias: '.siem-signals-another-1',
+        indexName: '.siem-signals-another-1-legacy',
+        space: 'another-1',
+      },
+      '.siem-signals-default-old-one': {
+        alias: '.siem-signals-default',
+        indexName: '.siem-signals-default-old-one',
+        space: 'default',
+      },
+    });
+  });
+
+  it('filters out .internal.alert indices', async () => {
+    esClient.indices.getAlias.mockResponseOnce({
+      '.siem-signals-default-old-one': {
+        aliases: {
+          '.siem-signals-default': {
+            is_write_index: false,
+          },
+        },
+      },
+      '.internal.alerts-security.alerts-another-2-000001': {
+        aliases: {
+          '.siem-signals-another-2': {
+            is_write_index: false,
+          },
+        },
+      },
+    });
+
+    const result = await getIndexAliasPerSpace({
+      esClient,
+      signalsIndex: '.siem-signals',
+      signalsAliasAllSpaces: '.siem-signals-*',
+    });
+
+    expect(result).toEqual({
+      '.siem-signals-default-old-one': {
+        alias: '.siem-signals-default',
+        indexName: '.siem-signals-default-old-one',
+        space: 'default',
+      },
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_index_alias_per_space.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { ElasticsearchClient } from '@kbn/core/server';
+
+interface IndexAlias {
+  alias: string;
+  space: string;
+  indexName: string;
+}
+
+/**
+ * Retrieves index, its alias and Kibana space
+ */
+export const getIndexAliasPerSpace = async ({
+  esClient,
+  signalsIndex,
+  signalsAliasAllSpaces,
+}: {
+  esClient: ElasticsearchClient;
+  signalsIndex: string;
+  signalsAliasAllSpaces: string;
+}): Promise<Record<string, IndexAlias>> => {
+  const response = await esClient.indices.getAlias(
+    {
+      name: signalsAliasAllSpaces,
+    },
+    { meta: true }
+  );
+
+  const indexAliasesMap = Object.keys(response.body).reduce<Record<string, IndexAlias>>(
+    (acc, indexName) => {
+      if (!indexName.startsWith('.internal.alerts-')) {
+        const alias = Object.keys(response.body[indexName].aliases)[0];
+
+        acc[indexName] = {
+          alias,
+          space: alias.replace(`${signalsIndex}-`, ''),
+          indexName,
+        };
+      }
+
+      return acc;
+    },
+    {}
+  );
+
+  return indexAliasesMap;
+};

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.test.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { IndicesGetIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
+import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
+
+describe('getIndexAliasPerSpace', () => {
+  let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
+
+  beforeEach(() => {
+    esClient = elasticsearchServiceMock.createElasticsearchClient();
+  });
+
+  it('returns latest index template version', async () => {
+    esClient.indices.getIndexTemplate.mockResponseOnce({
+      index_templates: [
+        { index_template: { version: 77 } },
+        { index_template: { version: 10 } },
+        { index_template: { version: 23 } },
+        { index_template: { version: 0 } },
+      ],
+    } as IndicesGetIndexTemplateResponse);
+
+    const version = await getLatestIndexTemplateVersion({
+      esClient,
+      name: '.siem-signals-*',
+    });
+
+    expect(version).toBe(77);
+  });
+
+  it('returns 0 if templates empty', async () => {
+    esClient.indices.getIndexTemplate.mockResponseOnce({
+      index_templates: [],
+    });
+
+    const version = await getLatestIndexTemplateVersion({
+      esClient,
+      name: '.siem-signals-*',
+    });
+
+    expect(version).toBe(0);
+  });
+
+  it('returns 0 if request fails', async () => {
+    esClient.indices.getIndexTemplate.mockRejectedValueOnce('Failure');
+
+    const version = await getLatestIndexTemplateVersion({
+      esClient,
+      name: '.siem-signals-*',
+    });
+
+    expect(version).toBe(0);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_latest_index_template_version.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { ElasticsearchClient } from '@kbn/core/server';
+
+/**
+ * Retrieves the latest version of index template
+ * There are can be multiple index templates across different Kibana spaces,
+ * so we get them all and return the latest(greatest) number
+ */
+export const getLatestIndexTemplateVersion = async ({
+  esClient,
+  name,
+}: {
+  esClient: ElasticsearchClient;
+  name: string;
+}): Promise<number> => {
+  let latestTemplateVersion: number;
+  try {
+    const response = await esClient.indices.getIndexTemplate({ name });
+    const versions = response.index_templates.map(
+      (template) => template.index_template.version ?? 0
+    );
+
+    latestTemplateVersion = versions.length ? Math.max(...versions) : 0;
+  } catch (e) {
+    latestTemplateVersion = 0;
+  }
+
+  return latestTemplateVersion;
+};