[Security Solution] Migrate remaining public Detection Engine APIs to…

… OpenAPI and code generation (#170330)

**Related to: elastic/security-team#7491

## Summary

Migrated remaining public Detection Engine endpoints to OpenAPI schema
and code generation:

- `POST /api/detection_engine/rules/_bulk_action`
- `GET /api/detection_engine/rules/_find`

 Also completed the migration of internal APIs:

- `GET /internal/detection_engine/rules/{ruleId}/execution/events`
- `GET /internal/detection_engine/rules/{ruleId}/execution/results`

### Other notable changes

- Changed how we compose Zod error messages for unions, see
`packages/kbn-zod-helpers/src/stringify_zod_error.ts`. Now we are trying
to list the validation errors of all union members but limiting the
total number of validation errors displayed to users.
- Addressed some remaining `TODO
https://github.com/elastic/security-team/issues/7491`
- Removed dependencies of the risk engine and timelines on detection
engine schemas
- Removed outdated legacy rule schemas that are no longer in use
- Added new schema helpers that work with query params:
`BooleanFromString` and `ArrayFromString`

![image](https://github.com/elastic/kibana/assets/1938181/f4898f11-04e2-4c82-bce9-e662ba78f724)

![image](https://github.com/elastic/kibana/assets/1938181/235234e7-c86c-49a1-b39f-6f9f8dc780e7)

packages/kbn-openapi-generator/src/template_service/templates/zod_operation_schema.handlebars

@@ -6,7 +6,7 @@
  */
 
 import { z } from "zod";
-import { requiredOptional, isValidDateMath } from "@kbn/zod-helpers"
+import { requiredOptional, isValidDateMath, ArrayFromString, BooleanFromString } from "@kbn/zod-helpers"
 
 {{> disclaimer}}
 

packages/kbn-openapi-generator/src/template_service/templates/zod_query_item.handlebars

@@ -19,23 +19,17 @@
 {{~/if~}}
 
 {{~#if (eq type "array")}}
-  z.preprocess(
-    (value: unknown) => (typeof value === "string") ? value === '' ? [] : value.split(",") : value,
-    z.array({{~> zod_schema_item items ~}})
-  )
+  ArrayFromString({{~> zod_schema_item items ~}})
   {{~#if minItems}}.min({{minItems}}){{/if~}}
   {{~#if maxItems}}.max({{maxItems}}){{/if~}}
   {{~#if (eq requiredBool false)}}.optional(){{/if~}}
   {{~#if (defined default)}}.default({{{toJSON default}}}){{/if~}}
 {{~/if~}}
 
 {{~#if (eq type "boolean")}}
-  z.preprocess(
-    (value: unknown) => (typeof value === "boolean") ? String(value) : value,
-    z.enum(["true", "false"])
-    {{~#if (defined default)}}.default("{{{toJSON default}}}"){{/if~}}
-    .transform((value) => value === "true")
-  )
+  BooleanFromString
+  {{~#if (eq requiredBool false)}}.optional(){{/if~}}
+  {{~#if (defined default)}}.default({{{toJSON default}}}){{/if~}}
 {{~/if~}}
 
 {{~#if (eq type "string")}}

packages/kbn-zod-helpers/index.ts

@@ -6,8 +6,11 @@
  * Side Public License, v 1.
  */
 
+export * from './src/array_from_string';
+export * from './src/boolean_from_string';
 export * from './src/expect_parse_error';
 export * from './src/expect_parse_success';
 export * from './src/is_valid_date_math';
 export * from './src/required_optional';
+export * from './src/safe_parse_result';
 export * from './src/stringify_zod_error';

packages/kbn-zod-helpers/src/array_from_string.test.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ArrayFromString } from './array_from_string';
+import * as z from 'zod';
+
+describe('ArrayFromString', () => {
+  const itemsSchema = z.string();
+
+  it('should return an array when input is a string', () => {
+    const result = ArrayFromString(itemsSchema).parse('a,b,c');
+    expect(result).toEqual(['a', 'b', 'c']);
+  });
+
+  it('should return an empty array when input is an empty string', () => {
+    const result = ArrayFromString(itemsSchema).parse('');
+    expect(result).toEqual([]);
+  });
+
+  it('should return the input as is when it is not a string', () => {
+    const input = ['a', 'b', 'c'];
+    const result = ArrayFromString(itemsSchema).parse(input);
+    expect(result).toEqual(input);
+  });
+
+  it('should throw an error when input is not a string or an array', () => {
+    expect(() => ArrayFromString(itemsSchema).parse(123)).toThrow();
+  });
+});

packages/kbn-zod-helpers/src/array_from_string.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as z from 'zod';
+
+/**
+ * This is a helper schema to convert comma separated strings to arrays. Useful
+ * for processing query params.
+ *
+ * @param schema Array items schema
+ * @returns Array schema that accepts a comma-separated string as input
+ */
+export function ArrayFromString<T extends z.ZodTypeAny>(schema: T) {
+  return z.preprocess(
+    (value: unknown) =>
+      typeof value === 'string' ? (value === '' ? [] : value.split(',')) : value,
+    z.array(schema)
+  );
+}

packages/kbn-zod-helpers/src/boolean_from_string.test.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { BooleanFromString } from './boolean_from_string';
+
+describe('BooleanFromString', () => {
+  it('should return true when input is "true"', () => {
+    expect(BooleanFromString.parse('true')).toBe(true);
+  });
+
+  it('should return false when input is "false"', () => {
+    expect(BooleanFromString.parse('false')).toBe(false);
+  });
+
+  it('should return true when input is true', () => {
+    expect(BooleanFromString.parse(true)).toBe(true);
+  });
+
+  it('should return false when input is false', () => {
+    expect(BooleanFromString.parse(false)).toBe(false);
+  });
+
+  it('should throw an error when input is not a boolean or "true" or "false"', () => {
+    expect(() => BooleanFromString.parse('not a boolean')).toThrow();
+    expect(() => BooleanFromString.parse(42)).toThrow();
+  });
+});

packages/kbn-zod-helpers/src/boolean_from_string.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import * as z from 'zod';
+
+/**
+ * This is a helper schema to convert a boolean string ("true" or "false") to a
+ * boolean. Useful for processing query params.
+ *
+ * Accepts "true" or "false" as strings, or a boolean.
+ */
+export const BooleanFromString = z
+  .enum(['true', 'false'])
+  .or(z.boolean())
+  .transform((value) => {
+    if (typeof value === 'boolean') {
+      return value;
+    }
+    return value === 'true';
+  });

packages/kbn-zod-helpers/src/expect_parse_success.ts

@@ -7,9 +7,14 @@
  */
 
 import type { SafeParseReturnType, SafeParseSuccess } from 'zod';
+import { stringifyZodError } from './stringify_zod_error';
 
 export function expectParseSuccess<Input, Output>(
   result: SafeParseReturnType<Input, Output>
 ): asserts result is SafeParseSuccess<Output> {
-  expect(result.success).toEqual(true);
+  if (!result.success) {
+    // We are throwing here instead of using assertions because we want to show
+    // the stringified error to assist with debugging.
+    throw new Error(`Expected parse success, got error: ${stringifyZodError(result.error)}`);
+  }
 }

packages/kbn-zod-helpers/src/safe_parse_result.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as z from 'zod';
+
+/**
+ * Safely parse a payload against a schema, returning the output or undefined.
+ * This method does not throw validation errors and is useful for validating
+ * optional objects when we don't care about errors.
+ *
+ * @param payload Schema payload
+ * @param schema Validation schema
+ * @returns Schema output or undefined
+ */
+export function safeParseResult<T extends z.ZodTypeAny>(
+  payload: unknown,
+  schema: T
+): T['_output'] | undefined {
+  const result = schema.safeParse(payload);
+  if (result.success) {
+    return result.data;
+  }
+}

packages/kbn-zod-helpers/src/stringify_zod_error.ts

@@ -6,16 +6,41 @@
  * Side Public License, v 1.
  */
 
-import { ZodError } from 'zod';
+import { ZodError, ZodIssue } from 'zod';
+
+const MAX_ERRORS = 5;
 
 export function stringifyZodError(err: ZodError<any>) {
-  return err.issues
-    .map((issue) => {
-      // If the path is empty, the error is for the root object
-      if (issue.path.length === 0) {
-        return issue.message;
-      }
-      return `${issue.path.join('.')}: ${issue.message}`;
-    })
-    .join(', ');
+  const errorMessages: string[] = [];
+
+  const issues = err.issues;
+
+  // Recursively traverse all issues
+  while (issues.length > 0) {
+    const issue = issues.shift()!;
+
+    // If the issue is an invalid union, we need to traverse all issues in the
+    // "unionErrors" array
+    if (issue.code === 'invalid_union') {
+      issues.push(...issue.unionErrors.flatMap((e) => e.issues));
+      continue;
+    }
+
+    errorMessages.push(stringifyIssue(issue));
+  }
+
+  const extraErrorCount = errorMessages.length - MAX_ERRORS;
+  if (extraErrorCount > 0) {
+    errorMessages.splice(MAX_ERRORS);
+    errorMessages.push(`and ${extraErrorCount} more`);
+  }
+
+  return errorMessages.join(', ');
+}
+
+function stringifyIssue(issue: ZodIssue) {
+  if (issue.path.length === 0) {
+    return issue.message;
+  }
+  return `${issue.path.join('.')}: ${issue.message}`;
 }

x-pack/plugins/security_solution/common/api/detection_engine/model/error_schema.mock.ts

@@ -4,9 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-// TODO https://github.com/elastic/security-team/issues/7491
-// eslint-disable-next-line no-restricted-imports
-import type { ErrorSchema } from './error_schema_legacy';
+import type { ErrorSchema } from './error_schema.gen';
 
 export const getErrorSchemaMock = (
   id: string = '819eded6-e9c8-445b-a647-519aea39e063'

x-pack/plugins/security_solution/common/api/detection_engine/model/index.ts

@@ -8,12 +8,8 @@
 export * from './alerts';
 export * from './rule_response_actions';
 export * from './rule_schema';
-// TODO https://github.com/elastic/security-team/issues/7491
-// eslint-disable-next-line no-restricted-imports
-export * from './error_schema_legacy';
-export * from './pagination';
+export * from './error_schema.gen';
+export * from './pagination.gen';
 export * from './schemas';
-// TODO https://github.com/elastic/security-team/issues/7491
-// eslint-disable-next-line no-restricted-imports
-export * from './sorting_legacy';
+export * from './sorting.gen';
 export * from './warning_schema.gen';

x-pack/plugins/security_solution/common/api/detection_engine/model/pagination.gen.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ */
+
+/**
+ * Page number
+ */
+export type Page = z.infer<typeof Page>;
+export const Page = z.number().int().min(1);
+
+/**
+ * Number of items per page
+ */
+export type PerPage = z.infer<typeof PerPage>;
+export const PerPage = z.number().int().min(0);
+
+export type PaginationResult = z.infer<typeof PaginationResult>;
+export const PaginationResult = z.object({
+  page: Page,
+  per_page: PerPage,
+  /**
+   * Total number of items
+   */
+  total: z.number().int().min(0),
+});

x-pack/plugins/security_solution/common/api/detection_engine/model/pagination.schema.yaml

@@ -0,0 +1,31 @@
+openapi: 3.0.0
+info:
+  title: Pagination Schema
+  version: 'not applicable'
+paths: {}
+components:
+  x-codegen-enabled: true
+  schemas:
+    Page:
+      type: integer
+      minimum: 1
+      description: Page number
+    PerPage:
+      type: integer
+      minimum: 0
+      description: Number of items per page
+    PaginationResult:
+      type: object
+      properties:
+        page:
+          $ref: '#/components/schemas/Page'
+        per_page:
+          $ref: '#/components/schemas/PerPage'
+        total:
+          type: integer
+          minimum: 0
+          description: Total number of items
+      required:
+        - page
+        - per_page
+        - total

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_response_actions/index.ts

@@ -5,7 +5,4 @@
  * 2.0.
  */
 
-// TODO https://github.com/elastic/security-team/issues/7491
-// eslint-disable-next-line no-restricted-imports
-export { RESPONSE_ACTION_TYPES, SUPPORTED_RESPONSE_ACTION_TYPES } from './response_actions_legacy';
 export * from './response_actions.gen';