[Security Solution] Data quality dashboard persistence

.eslintrc.js

@@ -1121,6 +1121,7 @@ module.exports = {
         'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
+        'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
       ],
       plugins: ['eslint-plugin-node', 'react'],
       env: {
@@ -1218,6 +1219,8 @@ module.exports = {
             'x-pack/plugins/security_solution_ess/**/*.{js,mjs,ts,tsx}',
             'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
             'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
+            'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}',
+            'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
           ],
           rules: {
             '@typescript-eslint/consistent-type-imports': 'error',

.github/CODEOWNERS

@@ -319,6 +319,7 @@ x-pack/packages/kbn-data-forge @elastic/obs-ux-management-team
 src/plugins/data @elastic/kibana-visualizations @elastic/kibana-data-discovery
 test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
+packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
 src/plugins/data_view_editor @elastic/kibana-data-discovery
 examples/data_view_field_editor_example @elastic/kibana-data-discovery
 src/plugins/data_view_field_editor @elastic/kibana-data-discovery

package.json

@@ -369,6 +369,7 @@
     "@kbn/data-plugin": "link:src/plugins/data",
     "@kbn/data-search-plugin": "link:test/plugin_functional/plugins/data_search",
     "@kbn/data-service": "link:packages/kbn-data-service",
+    "@kbn/data-stream-adapter": "link:packages/kbn-data-stream-adapter",
     "@kbn/data-view-editor-plugin": "link:src/plugins/data_view_editor",
     "@kbn/data-view-field-editor-example-plugin": "link:examples/data_view_field_editor_example",
     "@kbn/data-view-field-editor-plugin": "link:src/plugins/data_view_field_editor",

packages/kbn-data-stream-adapter/README.md

@@ -0,0 +1,69 @@
+# @kbn/data-stream-adapter
+
+Utility library for Elasticsearch data stream management.
+
+## DataStreamAdapter
+
+Manage single data streams. Example:
+
+```
+// Setup
+const dataStream = new DataStreamAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' });
+
+dataStream.setComponentTemplate({
+    name: 'awesome-component-template',
+    fieldMap: {
+        'awesome.field1: { type: 'keyword', required: true },
+        'awesome.nested.field2: { type: 'number', required: false },
+        // ...
+    },
+});
+
+dataStream.setIndexTemplate({
+    name: 'awesome-index-template',
+    componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
+    template: {
+        lifecycle: {
+            data_retention: '5d',
+        },
+    },
+});
+
+// Start
+await dataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and the data stream, or updates existing.
+```
+
+
+## DataStreamSpacesAdapter
+
+Manage data streams per space. Example:
+
+```
+// Setup
+const spacesDataStream = new DataStreamSpacesAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' });
+
+spacesDataStream.setComponentTemplate({
+    name: 'awesome-component-template',
+    fieldMap: {
+        'awesome.field1: { type: 'keyword', required: true },
+        'awesome.nested.field2: { type: 'number', required: false },
+        // ...
+    },
+});
+
+spacesDataStream.setIndexTemplate({
+    name: 'awesome-index-template',
+    componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
+    template: {
+        lifecycle: {
+            data_retention: '5d',
+        },
+    },
+});
+
+// Start
+await spacesDataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and updates existing data streams.
+
+// Create a space data stream on the fly
+await spacesDataStream.installSpace('space2'); // creates 'my-awesome-datastream-space2' data stream if it does not exist.
+```

packages/kbn-data-stream-adapter/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { DataStreamAdapter } from './src/data_stream_adapter';
+export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
+export { retryTransientEsErrors } from './src/retry_transient_es_errors';
+export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
+
+export type {
+  DataStreamAdapterParams,
+  SetComponentTemplateParams,
+  SetIndexTemplateParams,
+  InstallParams,
+} from './src/data_stream_adapter';
+export * from './src/field_maps/types';

packages/kbn-data-stream-adapter/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-data-stream-adapter'],
+};

packages/kbn-data-stream-adapter/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/data-stream-adapter",
+  "owner": "@elastic/security-threat-hunting-explore"
+}

packages/kbn-data-stream-adapter/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/data-stream-adapter",
+  "version": "1.0.0",
+  "description": "Utility library for Elasticsearch Data Stream management",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "private": true
+}