[OAS] Support lazy runtime types (#184000)

## Summary

Close #182910

Add the ability to declare recursive schemas. Updates
`@kbn/config-schema` to support recursive types. This design follows the
underlying pattern provided by Joi:
https://joi.dev/api/?v=17.13.0#linkref:

```ts
      const id = 'recursive';
      const recursiveSchema: Type<RecursiveType> = schema.object(
        {
          name: schema.string(),
          self: schema.lazy<RecursiveType>(id),
        },
        { meta: { id } }
      );
```

Since using the `.link` API we are also using `.id` which enables us to
leverage this mechanism OOTB with `joi-to-json` for OAS generation (thus
we could delete a lot of code there).

I chose to avoid using `id` originally because I thought it would be
simpler if we control more of the conversion in config-schema's case but
for recursive schemas and references I think this is a favourable trade
off. Open to other ideas!

---------

Co-authored-by: kibanamachine <[email protected]>

packages/core/status/core-status-server-internal/src/routes/status_response_schemas.ts

@@ -146,7 +146,7 @@ const fullStatusResponse: () => Type<Omit<StatusResponse, 'metrics'>> = () =>
     },
     {
       meta: {
-        id: 'core.status.response',
+        id: 'core_status_response',
         description: `Kibana's operational status as well as a detailed breakdown of plugin statuses indication of various loads (like event loop utilization and network traffic) at time of request.`,
       },
     }
@@ -163,7 +163,7 @@ const redactedStatusResponse = () =>
     },
     {
       meta: {
-        id: 'core.status.redactedResponse',
+        id: 'core_status_redactedResponse',
         description: `A minimal representation of Kibana's operational status.`,
       },
     }

packages/kbn-config-schema/index.ts

@@ -49,6 +49,7 @@ import {
   URIType,
   StreamType,
   UnionTypeOptions,
+  Lazy,
 } from './src/types';
 
 export type { AnyType, ConditionalType, TypeOf, Props, SchemaStructureEntry, NullableProps };
@@ -216,6 +217,13 @@ function conditional<A extends ConditionalTypeValue, B, C>(
   return new ConditionalType(leftOperand, rightOperand, equalType, notEqualType, options);
 }
 
+/**
+ * Useful for creating recursive schemas.
+ */
+function lazy<T>(id: string) {
+  return new Lazy<T>(id);
+}
+
 export const schema = {
   any,
   arrayOf,
@@ -226,6 +234,7 @@ export const schema = {
   contextRef,
   duration,
   ip,
+  lazy,
   literal,
   mapOf,
   maybe,
@@ -245,7 +254,6 @@ export type Schema = typeof schema;
 
 import {
   META_FIELD_X_OAS_ANY,
-  META_FIELD_X_OAS_REF_ID,
   META_FIELD_X_OAS_OPTIONAL,
   META_FIELD_X_OAS_DEPRECATED,
   META_FIELD_X_OAS_MAX_LENGTH,
@@ -255,7 +263,6 @@ import {
 
 export const metaFields = Object.freeze({
   META_FIELD_X_OAS_ANY,
-  META_FIELD_X_OAS_REF_ID,
   META_FIELD_X_OAS_OPTIONAL,
   META_FIELD_X_OAS_DEPRECATED,
   META_FIELD_X_OAS_MAX_LENGTH,

packages/kbn-config-schema/src/oas_meta_fields.ts

@@ -15,6 +15,5 @@ export const META_FIELD_X_OAS_MIN_LENGTH = 'x-oas-min-length' as const;
 export const META_FIELD_X_OAS_MAX_LENGTH = 'x-oas-max-length' as const;
 export const META_FIELD_X_OAS_GET_ADDITIONAL_PROPERTIES =
   'x-oas-get-additional-properties' as const;
-export const META_FIELD_X_OAS_REF_ID = 'x-oas-ref-id' as const;
 export const META_FIELD_X_OAS_DEPRECATED = 'x-oas-deprecated' as const;
 export const META_FIELD_X_OAS_ANY = 'x-oas-any-type' as const;

packages/kbn-config-schema/src/types/index.ts

@@ -40,3 +40,4 @@ export { URIType } from './uri_type';
 export { NeverType } from './never_type';
 export type { IpOptions } from './ip_type';
 export { IpType } from './ip_type';
+export { Lazy } from './lazy';

packages/kbn-config-schema/src/types/lazy.test.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Type, schema } from '../..';
+
+interface RecursiveType {
+  name: string;
+  self: undefined | RecursiveType;
+}
+
+// Test our recursive type inference
+{
+  const id = 'recursive';
+  // @ts-expect-error
+  const testObject: Type<RecursiveType> = schema.object(
+    {
+      name: schema.string(),
+      notSelf: schema.lazy<RecursiveType>(id), // this declaration should fail
+    },
+    { meta: { id } }
+  );
+}
+
+describe('lazy', () => {
+  const id = 'recursive';
+  const object = schema.object(
+    {
+      name: schema.string(),
+      self: schema.lazy<RecursiveType>(id),
+    },
+    { meta: { id } }
+  );
+
+  it('allows recursive runtime types to be defined', () => {
+    const self: RecursiveType = {
+      name: 'self1',
+      self: {
+        name: 'self2',
+        self: {
+          name: 'self3',
+          self: {
+            name: 'self4',
+            self: undefined,
+          },
+        },
+      },
+    };
+    const { value, error } = object.getSchema().validate(self);
+    expect(error).toBeUndefined();
+    expect(value).toEqual(self);
+  });
+
+  it('detects invalid recursive types as expected', () => {
+    const invalidSelf = {
+      name: 'self1',
+      self: {
+        name: 123,
+        self: undefined,
+      },
+    };
+
+    const { error, value } = object.getSchema().validate(invalidSelf);
+    expect(value).toEqual(invalidSelf);
+    expect(error?.message).toBe('expected value of type [string] but got [number]');
+  });
+});

packages/kbn-config-schema/src/types/lazy.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Type } from './type';
+import { internals } from '../internals';
+
+/**
+ * Use this type to construct recursive runtime schemas.
+ */
+export class Lazy<T> extends Type<T> {
+  constructor(id: string) {
+    super(internals.link(`#${id}`));
+  }
+}

packages/kbn-config-schema/src/types/object_type.test.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { get } from 'lodash';
 import { expectType } from 'tsd';
 import { schema } from '../..';
 import { TypeOf } from './object_type';
@@ -21,6 +22,16 @@ test('returns value by default', () => {
   expect(type.validate(value)).toEqual({ name: 'test' });
 });
 
+test('meta', () => {
+  const type = schema.object(
+    {
+      name: schema.string(),
+    },
+    { meta: { id: 'test_id' } }
+  );
+  expect(get(type.getSchema().describe(), 'flags.id')).toEqual('test_id');
+});
+
 test('returns empty object if undefined', () => {
   const type = schema.object({});
   expect(type.validate(undefined)).toEqual({});

packages/kbn-config-schema/src/types/object_type.ts

@@ -69,8 +69,16 @@ interface UnknownOptions {
   unknowns?: OptionsForUnknowns;
 }
 
+interface ObjectTypeOptionsMeta {
+  /**
+   * A string that uniquely identifies this schema. Used when generating OAS
+   * to create refs instead of inline schemas.
+   */
+  id?: string;
+}
+
 export type ObjectTypeOptions<P extends Props = any> = TypeOptions<ObjectResultType<P>> &
-  UnknownOptions;
+  UnknownOptions & { meta?: TypeOptions<ObjectResultType<P>>['meta'] & ObjectTypeOptionsMeta };
 
 export class ObjectType<P extends Props = any> extends Type<ObjectResultType<P>> {
   private props: P;
@@ -83,14 +91,18 @@ export class ObjectType<P extends Props = any> extends Type<ObjectResultType<P>>
     for (const [key, value] of Object.entries(props)) {
       schemaKeys[key] = value.getSchema();
     }
-    const schema = internals
+    let schema = internals
       .object()
       .keys(schemaKeys)
       .default()
       .optional()
       .unknown(unknowns === 'allow')
       .options({ stripUnknown: { objects: unknowns === 'ignore' } });
 
+    if (options.meta?.id) {
+      schema = schema.id(options.meta.id);
+    }
+
     super(schema, typeOptions);
     this.props = props;
     this.propSchemas = schemaKeys;

packages/kbn-config-schema/src/types/type.test.ts

@@ -9,7 +9,7 @@
 import { get } from 'lodash';
 import { internals } from '../internals';
 import { Type, TypeOptions } from './type';
-import { META_FIELD_X_OAS_REF_ID, META_FIELD_X_OAS_DEPRECATED } from '../oas_meta_fields';
+import { META_FIELD_X_OAS_DEPRECATED } from '../oas_meta_fields';
 
 class MyType extends Type<any> {
   constructor(opts: TypeOptions<any> = {}) {
@@ -20,12 +20,11 @@ class MyType extends Type<any> {
 describe('meta', () => {
   it('sets meta when provided', () => {
     const type = new MyType({
-      meta: { description: 'my description', id: 'foo', deprecated: true },
+      meta: { description: 'my description', deprecated: true },
     });
     const meta = type.getSchema().describe();
     expect(get(meta, 'flags.description')).toBe('my description');
-    expect(get(meta, `metas[0].${META_FIELD_X_OAS_REF_ID}`)).toBe('foo');
-    expect(get(meta, `metas[1].${META_FIELD_X_OAS_DEPRECATED}`)).toBe(true);
+    expect(get(meta, `metas[0].${META_FIELD_X_OAS_DEPRECATED}`)).toBe(true);
   });
 
   it('does not set meta when no provided', () => {

packages/kbn-config-schema/src/types/type.ts

@@ -7,7 +7,7 @@
  */
 
 import type { AnySchema, CustomValidator, ErrorReport } from 'joi';
-import { META_FIELD_X_OAS_DEPRECATED, META_FIELD_X_OAS_REF_ID } from '../oas_meta_fields';
+import { META_FIELD_X_OAS_DEPRECATED } from '../oas_meta_fields';
 import { SchemaTypeError, ValidationError } from '../errors';
 import { Reference } from '../references';
 
@@ -24,11 +24,6 @@ export interface TypeMeta {
    * Whether this field is deprecated.
    */
   deprecated?: boolean;
-  /**
-   * A string that uniquely identifies this schema. Used when generating OAS
-   * to create refs instead of inline schemas.
-   */
-  id?: string;
 }
 
 export interface TypeOptions<T> {
@@ -112,9 +107,6 @@ export abstract class Type<V> {
       if (options.meta.description) {
         schema = schema.description(options.meta.description);
       }
-      if (options.meta.id) {
-        schema = schema.meta({ [META_FIELD_X_OAS_REF_ID]: options.meta.id });
-      }
       if (options.meta.deprecated) {
         schema = schema.meta({ [META_FIELD_X_OAS_DEPRECATED]: true });
       }