Merge branch 'main' into adding-expanded-rows-to-pattern-analysis-table

.buildkite/scripts/pipelines/security_solution_ess/pipeline.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "Inside the security solution pipeline"

.eslintrc.js

@@ -1121,6 +1121,7 @@ module.exports = {
         'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
+        'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
       ],
       plugins: ['eslint-plugin-node', 'react'],
       env: {
@@ -1218,6 +1219,8 @@ module.exports = {
             'x-pack/plugins/security_solution_ess/**/*.{js,mjs,ts,tsx}',
             'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
             'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
+            'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}',
+            'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
           ],
           rules: {
             '@typescript-eslint/consistent-type-imports': 'error',

.github/CODEOWNERS

@@ -319,6 +319,7 @@ x-pack/packages/kbn-data-forge @elastic/obs-ux-management-team
 src/plugins/data @elastic/kibana-visualizations @elastic/kibana-data-discovery
 test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
+packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
 src/plugins/data_view_editor @elastic/kibana-data-discovery
 examples/data_view_field_editor_example @elastic/kibana-data-discovery
 src/plugins/data_view_field_editor @elastic/kibana-data-discovery

package.json

@@ -369,6 +369,7 @@
     "@kbn/data-plugin": "link:src/plugins/data",
     "@kbn/data-search-plugin": "link:test/plugin_functional/plugins/data_search",
     "@kbn/data-service": "link:packages/kbn-data-service",
+    "@kbn/data-stream-adapter": "link:packages/kbn-data-stream-adapter",
     "@kbn/data-view-editor-plugin": "link:src/plugins/data_view_editor",
     "@kbn/data-view-field-editor-example-plugin": "link:examples/data_view_field_editor_example",
     "@kbn/data-view-field-editor-plugin": "link:src/plugins/data_view_field_editor",
@@ -1002,7 +1003,6 @@
     "mapbox-gl-draw-rectangle-mode": "1.0.4",
     "maplibre-gl": "3.1.0",
     "markdown-it": "^12.3.2",
-    "md5": "^2.1.0",
     "mdast-util-to-hast": "10.2.0",
     "memoize-one": "^6.0.0",
     "mime": "^2.4.4",
@@ -1039,7 +1039,6 @@
     "pretty-ms": "6.0.0",
     "prop-types": "^15.8.1",
     "proxy-from-env": "1.0.0",
-    "puid": "1.0.7",
     "puppeteer": "21.5.2",
     "query-string": "^6.13.2",
     "rbush": "^3.0.1",

packages/kbn-alerts-ui-shared/src/alert_fields_table/index.tsx

@@ -17,6 +17,7 @@ import { css } from '@emotion/react';
 import React, { memo, useCallback, useMemo, useState } from 'react';
 import { Alert } from '@kbn/alerting-types';
 import { euiThemeVars } from '@kbn/ui-theme';
+import { EuiBasicTableColumn } from '@elastic/eui/src/components/basic_table/basic_table';
 
 export const search = {
   box: {
@@ -28,7 +29,7 @@ export const search = {
   },
 };
 
-const columns = [
+const columns: Array<EuiBasicTableColumn<AlertField>> = [
   {
     field: 'key',
     name: i18n.translate('alertsUIShared.alertFieldsTable.field', {
@@ -86,18 +87,46 @@ const useFieldBrowserPagination = () => {
   };
 };
 
+type AlertField = Exclude<
+  {
+    [K in keyof Alert]: { key: K; value: Alert[K] };
+  }[keyof Alert],
+  undefined
+>;
+
 export interface AlertFieldsTableProps {
+  /**
+   * The raw alert object
+   */
   alert: Alert;
+  /**
+   * A list of alert field keys to be shown in the table.
+   * When not defined, all the fields are shown.
+   */
+  fields?: Array<keyof Alert>;
 }
 
-export const AlertFieldsTable = memo(({ alert }: AlertFieldsTableProps) => {
+/**
+ * A paginated, filterable table to show alert object fields
+ */
+export const AlertFieldsTable = memo(({ alert, fields }: AlertFieldsTableProps) => {
   const { onTableChange, paginationTableProp } = useFieldBrowserPagination();
+  const items = useMemo(() => {
+    let _items = Object.entries(alert).map(
+      ([key, value]) =>
+        ({
+          key,
+          value,
+        } as AlertField)
+    );
+    if (fields?.length) {
+      _items = _items.filter((f) => fields.includes(f.key));
+    }
+    return _items;
+  }, [alert, fields]);
   return (
     <EuiInMemoryTable
-      items={Object.entries(alert).map(([key, value]) => ({
-        key,
-        value: Array.isArray(value) ? value?.[0] : value,
-      }))}
+      items={items}
       itemId="key"
       columns={columns}
       onTableChange={onTableChange}

packages/kbn-data-stream-adapter/README.md

@@ -0,0 +1,69 @@
+# @kbn/data-stream-adapter
+
+Utility library for Elasticsearch data stream management.
+
+## DataStreamAdapter
+
+Manage single data streams. Example:
+
+```
+// Setup
+const dataStream = new DataStreamAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' });
+
+dataStream.setComponentTemplate({
+    name: 'awesome-component-template',
+    fieldMap: {
+        'awesome.field1: { type: 'keyword', required: true },
+        'awesome.nested.field2: { type: 'number', required: false },
+        // ...
+    },
+});
+
+dataStream.setIndexTemplate({
+    name: 'awesome-index-template',
+    componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
+    template: {
+        lifecycle: {
+            data_retention: '5d',
+        },
+    },
+});
+
+// Start
+await dataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and the data stream, or updates existing.
+```
+
+
+## DataStreamSpacesAdapter
+
+Manage data streams per space. Example:
+
+```
+// Setup
+const spacesDataStream = new DataStreamSpacesAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' });
+
+spacesDataStream.setComponentTemplate({
+    name: 'awesome-component-template',
+    fieldMap: {
+        'awesome.field1: { type: 'keyword', required: true },
+        'awesome.nested.field2: { type: 'number', required: false },
+        // ...
+    },
+});
+
+spacesDataStream.setIndexTemplate({
+    name: 'awesome-index-template',
+    componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
+    template: {
+        lifecycle: {
+            data_retention: '5d',
+        },
+    },
+});
+
+// Start
+await spacesDataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and updates existing data streams.
+
+// Create a space data stream on the fly
+await spacesDataStream.installSpace('space2'); // creates 'my-awesome-datastream-space2' data stream if it does not exist.
+```

packages/kbn-data-stream-adapter/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { DataStreamAdapter } from './src/data_stream_adapter';
+export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
+export { retryTransientEsErrors } from './src/retry_transient_es_errors';
+export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
+
+export type {
+  DataStreamAdapterParams,
+  SetComponentTemplateParams,
+  SetIndexTemplateParams,
+  InstallParams,
+} from './src/data_stream_adapter';
+export * from './src/field_maps/types';

packages/kbn-data-stream-adapter/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-data-stream-adapter'],
+};

packages/kbn-data-stream-adapter/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/data-stream-adapter",
+  "owner": "@elastic/security-threat-hunting-explore"
+}

packages/kbn-data-stream-adapter/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/data-stream-adapter",
+  "version": "1.0.0",
+  "description": "Utility library for Elasticsearch Data Stream management",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "private": true
+}