create resources data stream

packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts

@@ -168,8 +168,6 @@ export async function createDataStream({
   esClient,
   name,
 }: CreateDataStreamParams): Promise<void> {
-  logger.info(`Creating data stream - ${name}`);
-
   // check if data stream exists
   let dataStreamExists = false;
   try {
@@ -190,6 +188,7 @@ export async function createDataStream({
     return;
   }
 
+  logger.info(`Creating data stream - ${name}`);
   try {
     await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });
   } catch (error) {

packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts

@@ -16,24 +16,25 @@ import {
   type InstallParams,
 } from './data_stream_adapter';
 
+type InstallSpace = (spaceId: string) => Promise<string>;
+
 export class DataStreamSpacesAdapter extends DataStreamAdapter {
   private installedSpaceDataStreamName: Map<string, Promise<string>>;
-  private _installSpace?: (spaceId: string) => Promise<string>;
+  private installSpacePromise?: Promise<InstallSpace>;
 
   constructor(private readonly prefix: string, options: DataStreamAdapterParams) {
     super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names
     this.installedSpaceDataStreamName = new Map();
   }
 
-  public async install({
-    logger,
-    esClient: esClientToResolve,
-    pluginStop$,
-    tasksTimeoutMs,
-  }: InstallParams) {
-    this.installed = true;
+  public async install(params: InstallParams): Promise<void> {
+    this.installSpacePromise = this._install(params);
+    await this.installSpacePromise;
+  }
 
-    const esClient = await esClientToResolve;
+  private async _install(params: InstallParams): Promise<InstallSpace> {
+    const { logger, pluginStop$, tasksTimeoutMs } = params;
+    const esClient = await params.esClient;
     const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
 
     // Install component templates in parallel
@@ -55,7 +56,11 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter {
     await Promise.all(
       this.indexTemplates.map((indexTemplate) =>
         installFn(
-          createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }),
+          createOrUpdateIndexTemplate({
+            template: indexTemplate,
+            esClient,
+            logger,
+          }),
           `create or update ${indexTemplate.name} index template`
         )
       )
@@ -73,27 +78,29 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter {
     );
 
     // define function to install data stream for spaces on demand
-    this._installSpace = async (spaceId: string) => {
+    const installSpace = async (spaceId: string) => {
       const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId);
       if (existingInstallPromise) {
         return existingInstallPromise;
       }
       const name = `${this.prefix}-${spaceId}`;
-      const installPromise = installFn(
+      const namePromise = installFn(
         createDataStream({ name, esClient, logger }),
         `create ${name} data stream`
       ).then(() => name);
-
-      this.installedSpaceDataStreamName.set(spaceId, installPromise);
-      return installPromise;
+      this.installedSpaceDataStreamName.set(spaceId, namePromise);
+      return namePromise;
     };
+    return installSpace;
   }
 
   public async installSpace(spaceId: string): Promise<string> {
-    if (!this._installSpace) {
+    if (!this.installSpacePromise) {
       throw new Error('Cannot installSpace before install');
     }
-    return this._installSpace(spaceId);
+    // Await for installSpacePromise to ensure installation is complete, since install call may not be awaited from the plugin
+    const installSpace = await this.installSpacePromise;
+    return installSpace(spaceId);
   }
 
   public async getInstalledSpaceName(spaceId: string): Promise<string | undefined> {

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts

@@ -10,7 +10,7 @@
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
  *
  * info:
- *   title: Common SIEM Migrations Attributes
+ *   title: SIEM Rule Migrations API common components
  *   version: not applicable
  */
 

x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml

@@ -1,6 +1,6 @@
 openapi: 3.0.3
 info:
-  title: Common SIEM Migrations Attributes
+  title: SIEM Rule Migrations API common components
   version: 'not applicable'
 paths: {}
 components:

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts

@@ -10,7 +10,7 @@
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
  *
  * info:
- *   title: SIEM Rules Migration API endpoint
+ *   title: SIEM Rules Migration API
  *   version: 1
  */
 

x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml

@@ -1,6 +1,6 @@
 openapi: 3.0.3
 info:
-  title: SIEM Rules Migration API endpoint
+  title: SIEM Rules Migration API
   version: '1'
 paths:
   /internal/siem_migrations/rules:

x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts

@@ -10,7 +10,7 @@
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
  *
  * info:
- *   title: Common Splunk Rules Attributes
+ *   title: SIEM Rule Migration common components
  *   version: not applicable
  */
 
@@ -178,3 +178,38 @@ export const RuleMigrationAllTaskStats = z.array(
     })
   )
 );
+
+/**
+ * The rule migration resource document object.
+ */
+export type RuleMigrationResource = z.infer<typeof RuleMigrationResource>;
+export const RuleMigrationResource = z.object({
+  /**
+   * The migration id for with the resource is associated.
+   */
+  migration_id: z.string(),
+  /**
+   * The type of the resource.
+   */
+  type: z.enum(['macro', 'list']),
+  /**
+   * The resource name identifier.
+   */
+  name: z.string(),
+  /**
+   * The resource content value.
+   */
+  content: z.string().optional(),
+  /**
+   * The resource arbitrary metadata.
+   */
+  metadata: z.object({}).optional(),
+  /**
+   * The moment of the last update
+   */
+  updated_at: z.string().optional(),
+  /**
+   * The user who last updated the resource
+   */
+  updated_by: z.string().optional(),
+});

x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml

@@ -1,6 +1,6 @@
 openapi: 3.0.3
 info:
-  title: Common Splunk Rules Attributes
+  title: SIEM Rule Migration common components
   version: 'not applicable'
 paths: {}
 components:
@@ -180,3 +180,36 @@ components:
               migration_id:
                 type: string
                 description: The migration id
+
+    RuleMigrationResource:
+      type: object
+      description: The rule migration resource document object.
+      required:
+        - migration_id
+        - type
+        - name
+      properties:
+        migration_id:
+          type: string
+          description: The migration id for with the resource is associated.
+        type:
+          type: string
+          description: The type of the resource.
+          enum:
+            - macro
+            - list
+        name:
+          type: string
+          description: The resource name identifier.
+        content:
+          type: string
+          description: The resource content value.
+        metadata:
+          type: object
+          description: The resource arbitrary metadata.
+        updated_at:
+          type: string
+          description: The moment of the last update
+        updated_by:
+          type: string
+          description: The user who last updated the resource

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts

@@ -29,15 +29,15 @@ export type RuleMigrationAllDataStats = Array<RuleMigrationDataStats & { migrati
 
 export class RuleMigrationsDataClient {
   constructor(
-    private dataStreamNamePromise: Promise<string>,
+    private indexNamePromises: Record<'rules' | 'resources', Promise<string>>,
     private currentUser: AuthenticatedUser,
     private esClient: ElasticsearchClient,
     private logger: Logger
   ) {}
 
   /** Indexes an array of rule migrations to be processed */
   async create(ruleMigrations: CreateRuleMigrationInput[]): Promise<void> {
-    const index = await this.dataStreamNamePromise;
+    const index = await this.indexNamePromises.rules;
     await this.esClient
       .bulk({
         refresh: 'wait_for',
@@ -59,7 +59,7 @@ export class RuleMigrationsDataClient {
 
   /** Retrieves an array of rule documents of a specific migrations */
   async getRules(migrationId: string): Promise<StoredRuleMigration[]> {
-    const index = await this.dataStreamNamePromise;
+    const index = await this.indexNamePromises.rules;
     const query = this.getFilterQuery(migrationId);
 
     const storedRuleMigrations = await this.esClient
@@ -79,7 +79,7 @@ export class RuleMigrationsDataClient {
    * - Multiple tasks should not process the same migration simultaneously.
    */
   async takePending(migrationId: string, size: number): Promise<StoredRuleMigration[]> {
-    const index = await this.dataStreamNamePromise;
+    const index = await this.indexNamePromises.rules;
     const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PENDING);
 
     const storedRuleMigrations = await this.esClient
@@ -117,7 +117,7 @@ export class RuleMigrationsDataClient {
   }
 
   /** Updates one rule migration with the provided data and sets the status to `completed` */
-  async saveFinished({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise<void> {
+  async saveCompleted({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise<void> {
     const doc = {
       ...ruleMigration,
       status: SiemMigrationStatus.COMPLETED,
@@ -150,7 +150,7 @@ export class RuleMigrationsDataClient {
 
   /** Updates all the rule migration with the provided id with status `processing` back to `pending` */
   async releaseProcessing(migrationId: string): Promise<void> {
-    const index = await this.dataStreamNamePromise;
+    const index = await this.indexNamePromises.rules;
     const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PROCESSING);
     const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` };
     await this.esClient.updateByQuery({ index, query, script, refresh: false }).catch((error) => {
@@ -161,7 +161,7 @@ export class RuleMigrationsDataClient {
 
   /** Updates all the rule migration with the provided id with status `processing` or `failed` back to `pending` */
   async releaseProcessable(migrationId: string): Promise<void> {
-    const index = await this.dataStreamNamePromise;
+    const index = await this.indexNamePromises.rules;
     const query = this.getFilterQuery(migrationId, [
       SiemMigrationStatus.PROCESSING,
       SiemMigrationStatus.FAILED,
@@ -175,7 +175,7 @@ export class RuleMigrationsDataClient {
 
   /** Retrieves the stats for the rule migrations with the provided id */
   async getStats(migrationId: string): Promise<RuleMigrationDataStats> {
-    const index = await this.dataStreamNamePromise;
+    const index = await this.indexNamePromises.rules;
     const query = this.getFilterQuery(migrationId);
     const aggregations = {
       pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } },
@@ -206,7 +206,7 @@ export class RuleMigrationsDataClient {
 
   /** Retrieves the stats for all the rule migrations aggregated by migration id */
   async getAllStats(): Promise<RuleMigrationAllDataStats> {
-    const index = await this.dataStreamNamePromise;
+    const index = await this.indexNamePromises.rules;
     const aggregations = {
       migrationIds: {
         terms: { field: 'migration_id' },