New Integration Assistant plugin

.eslintrc.js

@@ -1004,6 +1004,29 @@ module.exports = {
       },
     },
 
+    /**
+     * Integration assistant overrides
+     */
+    {
+      // front end and common typescript and javascript files only
+      files: [
+        'x-pack/plugins/integration_assistant/public/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/integration_assistant/common/**/*.{js,mjs,ts,tsx}',
+      ],
+      rules: {
+        'import/no-nodejs-modules': 'error',
+        'no-duplicate-imports': 'off',
+        '@typescript-eslint/no-duplicate-imports': 'error',
+        'no-restricted-imports': [
+          'error',
+          {
+            // prevents UI code from importing server side code and then webpack including it when doing builds
+            patterns: ['**/server/*'],
+          },
+        ],
+      },
+    },
+
     /**
      * ML overrides
      */
@@ -1068,6 +1091,7 @@ module.exports = {
       files: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{ts,tsx}',
         'x-pack/plugins/elastic_assistant/**/*.{ts,tsx}',
+        'x-pack/plugins/integration_assistant/**/*.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant-common/**/*.{ts,tsx}',
         'x-pack/packages/kbn-langchain/**/*.{ts,tsx}',
@@ -1082,6 +1106,7 @@ module.exports = {
       excludedFiles: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/elastic_assistant/**/*.{test,mock,test_helper}.{ts,tsx}',
+        'x-pack/plugins/integration_assistant/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant-common/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/packages/kbn-langchain/**/*.{test,mock,test_helper}.{ts,tsx}',
@@ -1102,6 +1127,7 @@ module.exports = {
       files: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{ts,tsx}',
         'x-pack/plugins/elastic_assistant/**/*.{ts,tsx}',
+        'x-pack/plugins/integration_assistant/**/*.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant-common/**/*.{ts,tsx}',
         'x-pack/packages/kbn-langchain/**/*.{ts,tsx}',
@@ -1141,6 +1167,7 @@ module.exports = {
       files: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/elastic_assistant/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/integration_assistant/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant-common/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/kbn-langchain/**/*.{js,mjs,ts,tsx}',

.github/CODEOWNERS

@@ -503,6 +503,7 @@ x-pack/plugins/observability_solution/infra @elastic/obs-ux-logs-team @elastic/o
 x-pack/plugins/ingest_pipelines @elastic/kibana-management
 src/plugins/input_control_vis @elastic/kibana-presentation
 src/plugins/inspector @elastic/kibana-presentation
+x-pack/plugins/integration_assistant @elastic/security-solution
 src/plugins/interactive_setup @elastic/kibana-security
 test/interactive_setup_api_integration/plugins/test_endpoints @elastic/kibana-security
 packages/kbn-interpreter @elastic/kibana-visualizations

docs/developer/plugin-list.asciidoc

@@ -638,6 +638,10 @@ the infrastructure monitoring use-case within Kibana.
 |The ingest_pipelines plugin provides Kibana support for Elasticsearch's ingest pipelines.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/integration_assistant/README.md[integrationAssistant]
+|Team owner: Security Integrations Scalability
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/investigate/README.md[investigate]
 |undefined
 

package.json

@@ -80,7 +80,7 @@
   "resolutions": {
     "**/@bazel/typescript/protobufjs": "6.11.4",
     "**/@hello-pangea/dnd": "16.6.0",
-    "**/@langchain/core": "0.1.53",
+    "**/@langchain/core": "0.2.3",
     "**/@types/node": "20.10.5",
     "**/@typescript-eslint/utils": "5.62.0",
     "**/chokidar": "^3.5.3",
@@ -540,6 +540,7 @@
     "@kbn/ingest-pipelines-plugin": "link:x-pack/plugins/ingest_pipelines",
     "@kbn/input-control-vis-plugin": "link:src/plugins/input_control_vis",
     "@kbn/inspector-plugin": "link:src/plugins/inspector",
+    "@kbn/integration-assistant-plugin": "link:x-pack/plugins/integration_assistant",
     "@kbn/interactive-setup-plugin": "link:src/plugins/interactive_setup",
     "@kbn/interactive-setup-test-endpoints-plugin": "link:test/interactive_setup_api_integration/plugins/test_endpoints",
     "@kbn/interpreter": "link:packages/kbn-interpreter",
@@ -924,9 +925,10 @@
     "@kbn/watcher-plugin": "link:x-pack/plugins/watcher",
     "@kbn/xstate-utils": "link:packages/kbn-xstate-utils",
     "@kbn/zod-helpers": "link:packages/kbn-zod-helpers",
-    "@langchain/community": "^0.0.44",
-    "@langchain/core": "^0.1.53",
-    "@langchain/openai": "^0.0.25",
+    "@langchain/community": "^0.2.4",
+    "@langchain/core": "0.2.3",
+    "@langchain/langgraph": "^0.0.21",
+    "@langchain/openai": "^0.0.34",
     "@langtrase/trace-attributes": "^3.0.8",
     "@langtrase/typescript-sdk": "^2.2.1",
     "@launchdarkly/node-server-sdk": "^9.4.5",
@@ -949,10 +951,10 @@
     "@paralleldrive/cuid2": "^2.2.2",
     "@reduxjs/toolkit": "1.9.7",
     "@slack/webhook": "^7.0.1",
-    "@smithy/eventstream-codec": "^2.0.12",
-    "@smithy/eventstream-serde-node": "^2.1.1",
-    "@smithy/types": "^2.9.1",
-    "@smithy/util-utf8": "^2.0.0",
+    "@smithy/eventstream-codec": "^3.0.0",
+    "@smithy/eventstream-serde-node": "^3.0.0",
+    "@smithy/types": "^3.0.0",
+    "@smithy/util-utf8": "^3.0.0",
     "@tanstack/react-query": "^4.29.12",
     "@tanstack/react-query-devtools": "^4.29.12",
     "@turf/along": "6.0.1",
@@ -1064,9 +1066,10 @@
     "jsonwebtoken": "^9.0.2",
     "jsts": "^1.6.2",
     "kea": "^2.6.0",
-    "langchain": "^0.1.30",
-    "langsmith": "^0.1.14",
+    "langchain": "0.2.3",
+    "langsmith": "^0.1.30",
     "launchdarkly-js-client-sdk": "^3.3.0",
+    "launchdarkly-node-server-sdk": "^7.0.3",
     "load-json-file": "^6.2.0",
     "lodash": "^4.17.21",
     "lru-cache": "^4.1.5",
@@ -1089,6 +1092,7 @@
     "node-forge": "^1.3.1",
     "nodemailer": "^6.9.9",
     "normalize-path": "^3.0.0",
+    "nunjucks": "^3.2.4",
     "object-hash": "^1.3.1",
     "object-path-immutable": "^3.1.1",
     "openai": "^4.24.1",
@@ -1501,6 +1505,7 @@
     "@types/node-forge": "^1.3.10",
     "@types/nodemailer": "^6.4.0",
     "@types/normalize-path": "^3.0.0",
+    "@types/nunjucks": "^3.2.6",
     "@types/object-hash": "^1.3.0",
     "@types/opn": "^5.1.0",
     "@types/ora": "^1.3.5",

tsconfig.base.json

@@ -1000,6 +1000,8 @@
       "@kbn/input-control-vis-plugin/*": ["src/plugins/input_control_vis/*"],
       "@kbn/inspector-plugin": ["src/plugins/inspector"],
       "@kbn/inspector-plugin/*": ["src/plugins/inspector/*"],
+      "@kbn/integration-assistant-plugin": ["x-pack/plugins/integration_assistant"],
+      "@kbn/integration-assistant-plugin/*": ["x-pack/plugins/integration_assistant/*"],
       "@kbn/interactive-setup-plugin": ["src/plugins/interactive_setup"],
       "@kbn/interactive-setup-plugin/*": ["src/plugins/interactive_setup/*"],
       "@kbn/interactive-setup-test-endpoints-plugin": ["test/interactive_setup_api_integration/plugins/test_endpoints"],

x-pack/packages/kbn-langchain/server/language_models/chat_openai.ts

@@ -27,6 +27,7 @@ export interface ActionsClientChatOpenAIParams {
   streaming?: boolean;
   traceId?: string;
   maxRetries?: number;
+  maxTokens?: number;
   model?: string;
   temperature?: number;
   signal?: AbortSignal;
@@ -75,9 +76,11 @@ export class ActionsClientChatOpenAI extends ChatOpenAI {
     streaming = true,
     temperature,
     timeout,
+    maxTokens,
   }: ActionsClientChatOpenAIParams) {
     super({
       maxRetries,
+      maxTokens,
       streaming,
       // matters only for the LangSmith logs (Metadata > Invocation Params), which are misleading if this is not set
       modelName: model ?? DEFAULT_OPEN_AI_MODEL,

x-pack/packages/kbn-langchain/server/language_models/simple_chat_model.ts

@@ -35,6 +35,7 @@ export interface CustomChatModelInput extends BaseChatModelParams {
   temperature?: number;
   request: KibanaRequest;
   streaming: boolean;
+  maxTokens?: number;
 }
 
 export class ActionsClientSimpleChatModel extends SimpleChatModel {
@@ -44,6 +45,7 @@ export class ActionsClientSimpleChatModel extends SimpleChatModel {
   #request: KibanaRequest;
   #traceId: string;
   #signal?: AbortSignal;
+  #maxTokens?: number;
   llmType: string;
   streaming: boolean;
   model?: string;
@@ -59,6 +61,7 @@ export class ActionsClientSimpleChatModel extends SimpleChatModel {
     temperature,
     signal,
     streaming,
+    maxTokens,
   }: CustomChatModelInput) {
     super({});
 
@@ -68,6 +71,7 @@ export class ActionsClientSimpleChatModel extends SimpleChatModel {
     this.#logger = logger;
     this.#signal = signal;
     this.#request = request;
+    this.#maxTokens = maxTokens;
     this.llmType = llmType ?? 'ActionsClientSimpleChatModel';
     this.model = model;
     this.temperature = temperature;
@@ -95,7 +99,7 @@ export class ActionsClientSimpleChatModel extends SimpleChatModel {
       throw new Error('No messages provided.');
     }
     const formattedMessages = [];
-    if (messages.length === 2) {
+    if (messages.length >= 2) {
       messages.forEach((message, i) => {
         if (typeof message.content !== 'string') {
           throw new Error('Multimodal messages are not supported.');
@@ -121,6 +125,7 @@ export class ActionsClientSimpleChatModel extends SimpleChatModel {
         subActionParams: {
           model: this.model,
           messages: formattedMessages,
+          maxTokens: this.#maxTokens,
           ...getDefaultArguments(this.llmType, this.temperature, options.stop),
         },
       },

x-pack/plugins/integration_assistant/README.md

@@ -0,0 +1,66 @@
+# Integration Assistant
+
+## Overview
+
+Team owner: Security Integrations Scalability
+
+This is a new Kibana plugin created to help users with automatically generating integration packages based on provided log samples and relevant information
+
+## Features
+
+Exposes 4 API's that can be consumed by any frontend plugin, which are:
+
+- ECS Mapping API
+- Categorization API
+- Related Fields API
+- Build Integration API
+- Optional Test Pipeline API (Used to update pipeline results if the ingest pipeline is changed by a user in the UI).
+
+## Development
+
+### Backend
+
+#### Overview
+
+The backend part of the plugin utilizes langraph extensively to parse the provided log samples and generate the integration package.
+
+One instance of langraph is created that will include one or more `nodes` in which each node represents a step in the integration package generation process.
+
+Each node links to a specific function, usually a `handler` specified in its own file under each graph folder that will be executed when the node is reached.
+
+#### Structure
+
+**Graphs**
+
+The graph components are split into logical parts and are placed in separate folders for each graph under the `./server/graphs` directory.
+
+Each graph folder needs to contains at least one `graph.ts`, which exports a function that returns the compiled graph object.
+
+Each exported graph function is then linked up to one or more API routes.
+
+**Routes**
+
+All routes are defined under `./server/routes` in its own file, and then included in the `./server/routes/register_routes.ts` file.
+
+**Integration Builder**
+
+The integration builder is the last step in the expected API flow (ECS Mapping -> Categorization -> Related Fields -> Integration Builder).
+With the provided package and data stream details, an optional logo and a list of sample logs, the API will build out the entire folder structure and files required for the integration package, archive it and return it as a `Buffer`.
+
+**Templates**
+
+Currently the templates are stored as `nunjucks` files as they were converted from `jinja2` templates, which use the exact same format. Longer term this will most likely be switched to the Kibana forked Handlebars templating engine.
+
+The templates are stored in the `./server/templates` directory and are used to generate the integration package files while running the Integration Builder API.
+
+One template (pipeline.yml.njk) is used by the ECS Mapping API to generate the boilerplate ingest pipeline structure we want to use for all generated integrations.
+
+## Tests
+
+All mocks/fixtures are placed in the top `./__jest__` directory of the plugin. If many mocks/fixtures are required, try to split them up into separate file(s).
+
+Tests can be run with:
+
+```bash
+node scripts/jest x-pack/plugins/integration_assistant/ --coverage
+```