[ML] Transforms: API schemas and integration tests (#75164) (#77357)

- Adds schema definitions to transform API endpoints and adds API integration tests.
- The type definitions based on the schema definitions can be used on the client side too.
- Adds apidoc documentation.

x-pack/plugins/ml/common/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { SearchResponse7 } from './types/es_client';

x-pack/plugins/ml/common/types/es_client.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SearchResponse, ShardsResponse } from 'elasticsearch';
+
+// The types specified in `@types/elasticsearch` are out of date and still have `total: number`.
+interface SearchResponse7Hits<T> {
+  hits: SearchResponse<T>['hits']['hits'];
+  max_score: number;
+  total: {
+    value: number;
+    relation: string;
+  };
+}
+export interface SearchResponse7<T = any> {
+  took: number;
+  timed_out: boolean;
+  _scroll_id?: string;
+  _shards: ShardsResponse;
+  hits: SearchResponse7Hits<T>;
+  aggregations?: any;
+}

x-pack/plugins/ml/public/application/components/data_grid/index.ts

@@ -19,7 +19,6 @@ export {
   DataGridItem,
   EsSorting,
   RenderCellValue,
-  SearchResponse7,
   UseDataGridReturnType,
   UseIndexDataReturnType,
 } from './types';

x-pack/plugins/ml/public/application/components/data_grid/types.ts

@@ -5,7 +5,6 @@
  */
 
 import { Dispatch, SetStateAction } from 'react';
-import { SearchResponse } from 'elasticsearch';
 
 import { EuiDataGridPaginationProps, EuiDataGridSorting, EuiDataGridColumn } from '@elastic/eui';
 
@@ -43,16 +42,6 @@ export type EsSorting = Dictionary<{
   order: 'asc' | 'desc';
 }>;
 
-// The types specified in `@types/elasticsearch` are out of date and still have `total: number`.
-export interface SearchResponse7 extends SearchResponse<any> {
-  hits: SearchResponse<any>['hits'] & {
-    total: {
-      value: number;
-      relation: string;
-    };
-  };
-}
-
 export interface UseIndexDataReturnType
   extends Pick<
     UseDataGridReturnType,

x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts

@@ -4,9 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import type { SearchResponse7 } from '../../../../common/types/es_client';
 import { extractErrorMessage } from '../../../../common/util/errors';
 
-import { EsSorting, SearchResponse7, UseDataGridReturnType } from '../../components/data_grid';
+import { EsSorting, UseDataGridReturnType } from '../../components/data_grid';
 import { ml } from '../../services/ml_api_service';
 
 import { isKeywordAndTextType } from '../common/fields';

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts

@@ -22,9 +22,9 @@ import {
   useDataGrid,
   useRenderCellValue,
   EsSorting,
-  SearchResponse7,
   UseIndexDataReturnType,
 } from '../../../../components/data_grid';
+import type { SearchResponse7 } from '../../../../../../common/types/es_client';
 import { extractErrorMessage } from '../../../../../../common/util/errors';
 import { INDEX_STATUS } from '../../../common/analytics';
 import { ml } from '../../../../services/ml_api_service';

x-pack/plugins/transform/common/api_schemas/audit_messages.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TransformMessage } from '../types/messages';
+
+export type GetTransformsAuditMessagesResponseSchema = TransformMessage[];

x-pack/plugins/transform/common/api_schemas/common.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import { TRANSFORM_STATE } from '../constants';
+
+export const transformIdsSchema = schema.arrayOf(
+  schema.object({
+    id: schema.string(),
+  })
+);
+
+export type TransformIdsSchema = TypeOf<typeof transformIdsSchema>;
+
+export const transformStateSchema = schema.oneOf([
+  schema.literal(TRANSFORM_STATE.ABORTING),
+  schema.literal(TRANSFORM_STATE.FAILED),
+  schema.literal(TRANSFORM_STATE.INDEXING),
+  schema.literal(TRANSFORM_STATE.STARTED),
+  schema.literal(TRANSFORM_STATE.STOPPED),
+  schema.literal(TRANSFORM_STATE.STOPPING),
+]);
+
+export const indexPatternTitleSchema = schema.object({
+  /** Title of the index pattern for which to return stats. */
+  indexPatternTitle: schema.string(),
+});
+
+export type IndexPatternTitleSchema = TypeOf<typeof indexPatternTitleSchema>;
+
+export const transformIdParamSchema = schema.object({
+  transformId: schema.string(),
+});
+
+export type TransformIdParamSchema = TypeOf<typeof transformIdParamSchema>;
+
+export interface ResponseStatus {
+  success: boolean;
+  error?: any;
+}
+
+export interface CommonResponseStatusSchema {
+  [key: string]: ResponseStatus;
+}

x-pack/plugins/transform/common/api_schemas/delete_transforms.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import { transformStateSchema, ResponseStatus } from './common';
+
+export const deleteTransformsRequestSchema = schema.object({
+  /**
+   * Delete Transform & Destination Index
+   */
+  transformsInfo: schema.arrayOf(
+    schema.object({
+      id: schema.string(),
+      state: transformStateSchema,
+    })
+  ),
+  deleteDestIndex: schema.maybe(schema.boolean()),
+  deleteDestIndexPattern: schema.maybe(schema.boolean()),
+  forceDelete: schema.maybe(schema.boolean()),
+});
+
+export type DeleteTransformsRequestSchema = TypeOf<typeof deleteTransformsRequestSchema>;
+
+export interface DeleteTransformStatus {
+  transformDeleted: ResponseStatus;
+  destIndexDeleted?: ResponseStatus;
+  destIndexPatternDeleted?: ResponseStatus;
+  destinationIndex?: string | undefined;
+}
+
+export interface DeleteTransformsResponseSchema {
+  [key: string]: DeleteTransformStatus;
+}

x-pack/plugins/transform/common/api_schemas/field_histograms.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export const fieldHistogramsRequestSchema = schema.object({
+  /** Query to match documents in the index. */
+  query: schema.any(),
+  /** The fields to return histogram data. */
+  fields: schema.arrayOf(schema.any()),
+  /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */
+  samplerShardSize: schema.number(),
+});
+
+export type FieldHistogramsRequestSchema = TypeOf<typeof fieldHistogramsRequestSchema>;
+export type FieldHistogramsResponseSchema = any[];

x-pack/plugins/transform/common/api_schemas/start_transforms.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TypeOf } from '@kbn/config-schema';
+
+import { transformIdsSchema, CommonResponseStatusSchema } from './common';
+
+export const startTransformsRequestSchema = transformIdsSchema;
+export type StartTransformsRequestSchema = TypeOf<typeof startTransformsRequestSchema>;
+export type StartTransformsResponseSchema = CommonResponseStatusSchema;

x-pack/plugins/transform/common/api_schemas/stop_transforms.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import { transformStateSchema, CommonResponseStatusSchema } from './common';
+
+export const stopTransformsRequestSchema = schema.arrayOf(
+  schema.object({
+    id: schema.string(),
+    state: transformStateSchema,
+  })
+);
+
+export type StopTransformsRequestSchema = TypeOf<typeof stopTransformsRequestSchema>;
+export type StopTransformsResponseSchema = CommonResponseStatusSchema;

x-pack/plugins/transform/common/api_schemas/transforms.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+import type { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common';
+
+import type { Dictionary } from '../types/common';
+import type { PivotAggDict } from '../types/pivot_aggs';
+import type { PivotGroupByDict } from '../types/pivot_group_by';
+import type { TransformId, TransformPivotConfig } from '../types/transform';
+
+import { transformStateSchema } from './common';
+
+// GET transforms
+export const getTransformsRequestSchema = schema.arrayOf(
+  schema.object({
+    id: schema.string(),
+    state: transformStateSchema,
+  })
+);
+
+export type GetTransformsRequestSchema = TypeOf<typeof getTransformsRequestSchema>;
+
+export interface GetTransformsResponseSchema {
+  count: number;
+  transforms: TransformPivotConfig[];
+}
+
+// schemas shared by parts of the preview, create and update endpoint
+export const destSchema = schema.object({
+  index: schema.string(),
+  pipeline: schema.maybe(schema.string()),
+});
+export const pivotSchema = schema.object({
+  group_by: schema.any(),
+  aggregations: schema.any(),
+});
+export const settingsSchema = schema.object({
+  max_page_search_size: schema.maybe(schema.number()),
+  // The default value is null, which disables throttling.
+  docs_per_second: schema.maybe(schema.nullable(schema.number())),
+});
+export const sourceSchema = schema.object({
+  index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
+  query: schema.maybe(schema.recordOf(schema.string(), schema.any())),
+});
+export const syncSchema = schema.object({
+  time: schema.object({
+    delay: schema.maybe(schema.string()),
+    field: schema.string(),
+  }),
+});
+
+// PUT transforms/{transformId}
+export const putTransformsRequestSchema = schema.object({
+  description: schema.maybe(schema.string()),
+  dest: destSchema,
+  frequency: schema.maybe(schema.string()),
+  pivot: pivotSchema,
+  settings: schema.maybe(settingsSchema),
+  source: sourceSchema,
+  sync: schema.maybe(syncSchema),
+});
+
+export interface PutTransformsRequestSchema extends TypeOf<typeof putTransformsRequestSchema> {
+  pivot: {
+    group_by: PivotGroupByDict;
+    aggregations: PivotAggDict;
+  };
+}
+
+interface TransformCreated {
+  transform: TransformId;
+}
+interface TransformCreatedError {
+  id: TransformId;
+  error: any;
+}
+export interface PutTransformsResponseSchema {
+  transformsCreated: TransformCreated[];
+  errors: TransformCreatedError[];
+}
+
+// POST transforms/_preview
+export const postTransformsPreviewRequestSchema = schema.object({
+  pivot: pivotSchema,
+  source: sourceSchema,
+});
+
+export interface PostTransformsPreviewRequestSchema
+  extends TypeOf<typeof postTransformsPreviewRequestSchema> {
+  pivot: {
+    group_by: PivotGroupByDict;
+    aggregations: PivotAggDict;
+  };
+}
+
+interface EsMappingType {
+  type: ES_FIELD_TYPES;
+}
+
+export type PreviewItem = Dictionary<any>;
+export type PreviewData = PreviewItem[];
+export type PreviewMappingsProperties = Dictionary<EsMappingType>;
+
+export interface PostTransformsPreviewResponseSchema {
+  generated_dest_index: {
+    mappings: {
+      _meta: {
+        _transform: {
+          transform: string;
+          version: { create: string };
+          creation_date_in_millis: number;
+        };
+        created_by: string;
+      };
+      properties: PreviewMappingsProperties;
+    };
+    settings: { index: { number_of_shards: string; auto_expand_replicas: string } };
+    aliases: Record<string, any>;
+  };
+  preview: PreviewData;
+}

x-pack/plugins/transform/common/api_schemas/transforms_stats.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TypeOf } from '@kbn/config-schema';
+
+import { TransformStats } from '../types/transform_stats';
+
+import { getTransformsRequestSchema } from './transforms';
+
+export const getTransformsStatsRequestSchema = getTransformsRequestSchema;
+
+export type GetTransformsRequestSchema = TypeOf<typeof getTransformsStatsRequestSchema>;
+
+export interface GetTransformsStatsResponseSchema {
+  node_failures?: object;
+  count: number;
+  transforms: TransformStats[];
+}