[IndexAdapter] Extract index-adapter package from data-stream-adapter (…

…elastic#199575) ## Summary Extracts `IndexAdapter` from `DataStreamAdapter` and `IndexPatternAdapter` from `DataStreamSpaceAdapter`. There are no breaking changes for the _data-stream-adapter_ package; the behavior of both the `DataStreamAdapter` and `DataStreamSpaceAdapter` remains unchanged. The new _index-adapter_ package exports `IndexAdapter` and `IndexPatternAdapter` to manage individual indices without using data streams. This is needed for SIEM rule migrations. --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Elastic Machine <[email protected]>
tkajtoch · Nov 12, 2024 · c200419 · c200419
1 parent 0a5ffb5
commit c200419
Show file tree

Hide file tree

Showing 35 changed files with 860 additions and 222 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -324,7 +324,7 @@ packages/kbn-custom-icons @elastic/obs-ux-logs-team
 packages/kbn-custom-integrations @elastic/obs-ux-logs-team
 packages/kbn-cypress-config @elastic/kibana-operations
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
-packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
+packages/kbn-data-stream-adapter @elastic/security-threat-hunting
 packages/kbn-data-view-utils @elastic/kibana-data-discovery
 packages/kbn-datemath @elastic/kibana-data-discovery
 packages/kbn-dev-cli-errors @elastic/kibana-operations
@@ -380,6 +380,7 @@ packages/kbn-i18n @elastic/kibana-core
 packages/kbn-i18n-react @elastic/kibana-core
 packages/kbn-import-locator @elastic/kibana-operations
 packages/kbn-import-resolver @elastic/kibana-operations
+packages/kbn-index-adapter @elastic/security-threat-hunting
 packages/kbn-interpreter @elastic/kibana-visualizations
 packages/kbn-investigation-shared @elastic/obs-ux-management-team
 packages/kbn-io-ts-utils @elastic/obs-knowledge-team

diff --git a/package.json b/package.json
@@ -567,6 +567,7 @@
     "@kbn/i18n-react": "link:packages/kbn-i18n-react",
     "@kbn/iframe-embedded-plugin": "link:x-pack/test/functional_embedded/plugins/iframe_embedded",
     "@kbn/image-embeddable-plugin": "link:src/plugins/image_embeddable",
+    "@kbn/index-adapter": "link:packages/kbn-index-adapter",
     "@kbn/index-lifecycle-management-common-shared": "link:x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared",
     "@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
     "@kbn/index-management-plugin": "link:x-pack/plugins/index_management",

diff --git a/packages/kbn-data-stream-adapter/index.ts b/packages/kbn-data-stream-adapter/index.ts
@@ -9,13 +9,13 @@
 
 export { DataStreamAdapter } from './src/data_stream_adapter';
 export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
-export { retryTransientEsErrors } from './src/retry_transient_es_errors';
-export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
 
+export { retryTransientEsErrors, ecsFieldMap } from '@kbn/index-adapter';
 export type {
-  DataStreamAdapterParams,
   SetComponentTemplateParams,
   SetIndexTemplateParams,
   InstallParams,
-} from './src/data_stream_adapter';
-export * from './src/field_maps/types';
+  EcsFieldMap,
+} from '@kbn/index-adapter';
+
+export * from '@kbn/index-adapter/src/field_maps/types';
diff --git a/packages/kbn-data-stream-adapter/kibana.jsonc b/packages/kbn-data-stream-adapter/kibana.jsonc
@@ -1,5 +1,6 @@
 {
-  "type": "shared-common",
+  "type": "shared-server",
   "id": "@kbn/data-stream-adapter",
-  "owner": "@elastic/security-threat-hunting-explore"
+  "owner": "@elastic/security-threat-hunting",
+  "visibility": "shared"
 }
diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts
@@ -136,10 +136,11 @@ describe('createOrUpdateDataStream', () => {
   it(`should create data stream if not exists`, async () => {
     esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });
 
-    await createDataStream({
+    await createOrUpdateDataStream({
       esClient,
       logger,
       name,
+      totalFieldsLimit,
     });
 
     expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });

diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts
@@ -11,7 +11,7 @@ import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
 import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { Logger, ElasticsearchClient } from '@kbn/core/server';
 import { get } from 'lodash';
-import { retryTransientEsErrors } from './retry_transient_es_errors';
+import { retryTransientEsErrors } from '@kbn/index-adapter';
 
 interface UpdateIndexMappingsOpts {
   logger: Logger;
@@ -168,7 +168,7 @@ export async function createDataStream({
   esClient,
   name,
 }: CreateDataStreamParams): Promise<void> {
-  logger.info(`Creating data stream - ${name}`);
+  logger.debug(`Checking data stream exists - ${name}`);
 
   // check if data stream exists
   let dataStreamExists = false;
@@ -189,6 +189,7 @@ export async function createDataStream({
   if (dataStreamExists) {
     return;
   }
+  logger.info(`Installing data stream - ${name}`);
 
   try {
     await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });

diff --git a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts
@@ -7,145 +7,22 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import type {
-  ClusterPutComponentTemplateRequest,
-  IndicesIndexSettings,
-  IndicesPutIndexTemplateIndexTemplateMapping,
-  IndicesPutIndexTemplateRequest,
-} from '@elastic/elasticsearch/lib/api/types';
-import type { Logger, ElasticsearchClient } from '@kbn/core/server';
-import type { Subject } from 'rxjs';
-import type { FieldMap } from './field_maps/types';
-import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
+import { IndexAdapter, SetIndexTemplateParams, type InstallParams } from '@kbn/index-adapter';
 import { createOrUpdateDataStream } from './create_or_update_data_stream';
-import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
-import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
-import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils';
-
-export interface DataStreamAdapterParams {
-  kibanaVersion: string;
-  totalFieldsLimit?: number;
-}
-export interface SetComponentTemplateParams {
-  name: string;
-  fieldMap: FieldMap;
-  settings?: IndicesIndexSettings;
-  dynamic?: 'strict' | boolean;
-}
-export interface SetIndexTemplateParams {
-  name: string;
-  componentTemplateRefs?: string[];
-  namespace?: string;
-  template?: IndicesPutIndexTemplateIndexTemplateMapping;
-  hidden?: boolean;
-}
-
-export interface GetInstallFnParams {
-  logger: Logger;
-  pluginStop$: Subject<void>;
-  tasksTimeoutMs?: number;
-}
-export interface InstallParams {
-  logger: Logger;
-  esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
-  pluginStop$: Subject<void>;
-  tasksTimeoutMs?: number;
-}
-
-const DEFAULT_FIELDS_LIMIT = 2500;
-
-export class DataStreamAdapter {
-  protected readonly kibanaVersion: string;
-  protected readonly totalFieldsLimit: number;
-  protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
-  protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
-  protected installed: boolean;
-
-  constructor(protected readonly name: string, options: DataStreamAdapterParams) {
-    this.installed = false;
-    this.kibanaVersion = options.kibanaVersion;
-    this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
-  }
-
-  public setComponentTemplate(params: SetComponentTemplateParams) {
-    if (this.installed) {
-      throw new Error('Cannot set component template after install');
-    }
-    this.componentTemplates.push(getComponentTemplate(params));
-  }
 
+export class DataStreamAdapter extends IndexAdapter {
   public setIndexTemplate(params: SetIndexTemplateParams) {
-    if (this.installed) {
-      throw new Error('Cannot set index template after install');
-    }
-    this.indexTemplates.push(
-      getIndexTemplate({
-        ...params,
-        indexPatterns: [this.name],
-        kibanaVersion: this.kibanaVersion,
-        totalFieldsLimit: this.totalFieldsLimit,
-      })
-    );
-  }
-
-  protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
-    return async (promise: Promise<void>, description?: string): Promise<void> => {
-      try {
-        await installWithTimeout({
-          installFn: () => promise,
-          description,
-          timeoutMs: tasksTimeoutMs,
-          pluginStop$,
-        });
-      } catch (err) {
-        if (err instanceof InstallShutdownError) {
-          logger.info(err.message);
-        } else {
-          throw err;
-        }
-      }
-    };
+    super.setIndexTemplate({ ...params, isDataStream: true });
   }
 
-  public async install({
-    logger,
-    esClient: esClientToResolve,
-    pluginStop$,
-    tasksTimeoutMs,
-  }: InstallParams) {
+  public async install(params: InstallParams) {
     this.installed = true;
+    const { logger, pluginStop$, tasksTimeoutMs } = params;
+    const esClient = await params.esClient;
 
-    const esClient = await esClientToResolve;
-    const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
-
-    // Install component templates in parallel
-    await Promise.all(
-      this.componentTemplates.map((componentTemplate) =>
-        installFn(
-          createOrUpdateComponentTemplate({
-            template: componentTemplate,
-            esClient,
-            logger,
-            totalFieldsLimit: this.totalFieldsLimit,
-          }),
-          `${componentTemplate.name} component template`
-        )
-      )
-    );
+    await this.installTemplates(params);
 
-    // Install index templates in parallel
-    await Promise.all(
-      this.indexTemplates.map((indexTemplate) =>
-        installFn(
-          createOrUpdateIndexTemplate({
-            template: indexTemplate,
-            esClient,
-            logger,
-          }),
-          `${indexTemplate.name} index template`
-        )
-      )
-    );
+    const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
 
     // create data stream when everything is ready
     await installFn(

diff --git a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts
@@ -7,59 +7,26 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
-import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
-import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
 import {
-  DataStreamAdapter,
-  type DataStreamAdapterParams,
+  IndexPatternAdapter,
+  type SetIndexTemplateParams,
   type InstallParams,
-} from './data_stream_adapter';
-
-export class DataStreamSpacesAdapter extends DataStreamAdapter {
-  private installedSpaceDataStreamName: Map<string, Promise<string>>;
-  private _installSpace?: (spaceId: string) => Promise<string>;
+  type InstallIndex,
+} from '@kbn/index-adapter';
+import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
 
-  constructor(private readonly prefix: string, options: DataStreamAdapterParams) {
-    super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names
-    this.installedSpaceDataStreamName = new Map();
+export class DataStreamSpacesAdapter extends IndexPatternAdapter {
+  public setIndexTemplate(params: SetIndexTemplateParams) {
+    super.setIndexTemplate({ ...params, isDataStream: true });
   }
 
-  public async install({
-    logger,
-    esClient: esClientToResolve,
-    pluginStop$,
-    tasksTimeoutMs,
-  }: InstallParams) {
-    this.installed = true;
+  protected async _install(params: InstallParams): Promise<InstallIndex> {
+    const { logger, pluginStop$, tasksTimeoutMs } = params;
 
-    const esClient = await esClientToResolve;
-    const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
+    await this.installTemplates(params);
 
-    // Install component templates in parallel
-    await Promise.all(
-      this.componentTemplates.map((componentTemplate) =>
-        installFn(
-          createOrUpdateComponentTemplate({
-            template: componentTemplate,
-            esClient,
-            logger,
-            totalFieldsLimit: this.totalFieldsLimit,
-          }),
-          `create or update ${componentTemplate.name} component template`
-        )
-      )
-    );
-
-    // Install index templates in parallel
-    await Promise.all(
-      this.indexTemplates.map((indexTemplate) =>
-        installFn(
-          createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }),
-          `create or update ${indexTemplate.name} index template`
-        )
-      )
-    );
+    const esClient = await params.esClient;
+    const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
 
     // Update existing space data streams
     await installFn(
@@ -72,31 +39,21 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter {
       `update space data streams`
     );
 
-    // define function to install data stream for spaces on demand
-    this._installSpace = async (spaceId: string) => {
-      const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId);
-      if (existingInstallPromise) {
-        return existingInstallPromise;
-      }
-      const name = `${this.prefix}-${spaceId}`;
-      const installPromise = installFn(
-        createDataStream({ name, esClient, logger }),
-        `create ${name} data stream`
-      ).then(() => name);
-
-      this.installedSpaceDataStreamName.set(spaceId, installPromise);
-      return installPromise;
-    };
+    // define function to install data stream on demand
+    return async (name: string) =>
+      installFn(createDataStream({ name, esClient, logger }), `create ${name} data stream`);
   }
 
+  /**
+   * Method to create the data stream for a given space ID.
+   * It resolves with the full data stream name.
+   */
   public async installSpace(spaceId: string): Promise<string> {
-    if (!this._installSpace) {
-      throw new Error('Cannot installSpace before install');
-    }
-    return this._installSpace(spaceId);
+    await this.createIndex(spaceId);
+    return this.getIndexName(spaceId);
   }
 
   public async getInstalledSpaceName(spaceId: string): Promise<string | undefined> {
-    return this.installedSpaceDataStreamName.get(spaceId);
+    return this.getInstalledIndexName(spaceId);
   }
 }
diff --git a/packages/kbn-data-stream-adapter/tsconfig.json b/packages/kbn-data-stream-adapter/tsconfig.json
@@ -5,18 +5,14 @@
     "types": [
       "jest",
       "node",
-      "react",
-      "@emotion/react/types/css-prop",
-      "@testing-library/jest-dom",
-      "@testing-library/react"
     ]
   },
-  "include": ["**/*.ts", "**/*.tsx"],
+  "include": ["**/*.ts"],
   "kbn_references": [
     "@kbn/core",
-    "@kbn/std",
-    "@kbn/safer-lodash-set",
-    "@kbn/logging-mocks",
+    "@kbn/index-adapter",
+  ],
+  "exclude": [
+    "target/**/*"
   ],
-  "exclude": ["target/**/*"]
 }