diff --git a/x-pack/plugins/ml/common/types/data_frame_analytics.ts b/x-pack/plugins/ml/common/types/data_frame_analytics.ts index 5ba7f9c191a7f..f0aac75047585 100644 --- a/x-pack/plugins/ml/common/types/data_frame_analytics.ts +++ b/x-pack/plugins/ml/common/types/data_frame_analytics.ts @@ -5,7 +5,77 @@ */ import { CustomHttpResponseOptions, ResponseError } from 'kibana/server'; + export interface DeleteDataFrameAnalyticsWithIndexStatus { success: boolean; error?: CustomHttpResponseOptions; } + +export type IndexName = string; +export type DataFrameAnalyticsId = string; + +export interface OutlierAnalysis { + [key: string]: {}; + + outlier_detection: {}; +} + +interface Regression { + dependent_variable: string; + training_percent?: number; + num_top_feature_importance_values?: number; + prediction_field_name?: string; +} + +interface Classification { + dependent_variable: string; + training_percent?: number; + num_top_classes?: string; + num_top_feature_importance_values?: number; + prediction_field_name?: string; +} + +export interface RegressionAnalysis { + [key: string]: Regression; + + regression: Regression; +} + +export interface ClassificationAnalysis { + [key: string]: Classification; + + classification: Classification; +} + +interface GenericAnalysis { + [key: string]: Record; +} + +export type AnalysisConfig = + | OutlierAnalysis + | RegressionAnalysis + | ClassificationAnalysis + | GenericAnalysis; + +export interface DataFrameAnalyticsConfig { + id: DataFrameAnalyticsId; + description?: string; + dest: { + index: IndexName; + results_field: string; + }; + source: { + index: IndexName | IndexName[]; + query?: any; + }; + analysis: AnalysisConfig; + analyzed_fields: { + includes: string[]; + excludes: string[]; + }; + model_memory_limit: string; + max_num_threads?: number; + create_time: number; + version: string; + allow_lazy_start?: boolean; +} diff --git a/x-pack/plugins/ml/common/types/inference.ts b/x-pack/plugins/ml/common/types/inference.ts new file mode 100644 index 0000000000000..c70ee264e6fc8 --- /dev/null +++ b/x-pack/plugins/ml/common/types/inference.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DataFrameAnalyticsConfig } from './data_frame_analytics'; + +export interface IngestStats { + count: number; + time_in_millis: number; + current: number; + failed: number; +} + +export interface TrainedModelStat { + model_id: string; + pipeline_count: number; + inference_stats?: { + failure_count: number; + inference_count: number; + cache_miss_count: number; + missing_all_fields_count: number; + timestamp: number; + }; + ingest?: { + total: IngestStats; + pipelines: Record< + string, + IngestStats & { + processors: Array< + Record< + string, + { + // TODO use type from ingest_pipelines plugin + type: string; + stats: IngestStats; + } + > + >; + } + >; + }; +} + +export interface ModelConfigResponse { + created_by: string; + create_time: string; + default_field_map: Record; + estimated_heap_memory_usage_bytes: number; + estimated_operations: number; + license_level: string; + metadata?: + | { + analytics_config: DataFrameAnalyticsConfig; + input: any; + } + | Record; + model_id: string; + tags: string; + version: string; + inference_config?: Record; + pipelines?: Record | null; +} + +export interface PipelineDefinition { + processors?: Array>; + description?: string; +} + +export interface ModelPipelines { + model_id: string; + pipelines: Record; +} + +/** + * Get inference response from the ES endpoint + */ +export interface InferenceConfigResponse { + trained_model_configs: ModelConfigResponse[]; +} diff --git a/x-pack/plugins/ml/public/application/components/stats_bar/index.ts b/x-pack/plugins/ml/public/application/components/stats_bar/index.ts index 597975d0b150b..c8023b13cf74e 100644 --- a/x-pack/plugins/ml/public/application/components/stats_bar/index.ts +++ b/x-pack/plugins/ml/public/application/components/stats_bar/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats } from './stats_bar'; +export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats, ModelsBarStats } from './stats_bar'; diff --git a/x-pack/plugins/ml/public/application/components/stats_bar/stats_bar.tsx b/x-pack/plugins/ml/public/application/components/stats_bar/stats_bar.tsx index 0bd33a8c99f49..c4d64e48151b3 100644 --- a/x-pack/plugins/ml/public/application/components/stats_bar/stats_bar.tsx +++ b/x-pack/plugins/ml/public/application/components/stats_bar/stats_bar.tsx @@ -23,7 +23,11 @@ export interface AnalyticStatsBarStats extends Stats { stopped: StatsBarStat; } -export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats; +export interface ModelsBarStats { + total: StatsBarStat; +} + +export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats | ModelsBarStats; type StatsKey = keyof StatsBarStats; interface StatsBarProps { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts index 561e8642772f0..8ad861e616b7a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts @@ -5,18 +5,21 @@ */ import { useEffect } from 'react'; -import { BehaviorSubject } from 'rxjs'; -import { filter, distinctUntilChanged } from 'rxjs/operators'; -import { Subscription } from 'rxjs'; +import { BehaviorSubject, Subscription } from 'rxjs'; +import { distinctUntilChanged, filter } from 'rxjs/operators'; import { cloneDeep } from 'lodash'; import { ml } from '../../services/ml_api_service'; import { Dictionary } from '../../../../common/types/common'; import { getErrorMessage } from '../../../../common/util/errors'; import { SavedSearchQuery } from '../../contexts/ml'; +import { + AnalysisConfig, + ClassificationAnalysis, + OutlierAnalysis, + RegressionAnalysis, +} from '../../../../common/types/data_frame_analytics'; -export type IndexName = string; export type IndexPattern = string; -export type DataFrameAnalyticsId = string; export enum ANALYSIS_CONFIG_TYPE { OUTLIER_DETECTION = 'outlier_detection', @@ -46,34 +49,6 @@ export enum OUTLIER_ANALYSIS_METHOD { DISTANCE_KNN = 'distance_knn', } -interface OutlierAnalysis { - [key: string]: {}; - outlier_detection: {}; -} - -interface Regression { - dependent_variable: string; - training_percent?: number; - num_top_feature_importance_values?: number; - prediction_field_name?: string; -} -export interface RegressionAnalysis { - [key: string]: Regression; - regression: Regression; -} - -interface Classification { - dependent_variable: string; - training_percent?: number; - num_top_classes?: string; - num_top_feature_importance_values?: number; - prediction_field_name?: string; -} -export interface ClassificationAnalysis { - [key: string]: Classification; - classification: Classification; -} - export interface LoadExploreDataArg { filterByIsTraining?: boolean; searchQuery: SavedSearchQuery; @@ -165,22 +140,12 @@ export interface ClassificationEvaluateResponse { }; } -interface GenericAnalysis { - [key: string]: Record; -} - interface LoadEvaluateResult { success: boolean; eval: RegressionEvaluateResponse | ClassificationEvaluateResponse | null; error: string | null; } -type AnalysisConfig = - | OutlierAnalysis - | RegressionAnalysis - | ClassificationAnalysis - | GenericAnalysis; - export const getAnalysisType = (analysis: AnalysisConfig): string => { const keys = Object.keys(analysis); @@ -342,29 +307,6 @@ export interface UpdateDataFrameAnalyticsConfig { max_num_threads?: number; } -export interface DataFrameAnalyticsConfig { - id: DataFrameAnalyticsId; - description?: string; - dest: { - index: IndexName; - results_field: string; - }; - source: { - index: IndexName | IndexName[]; - query?: any; - }; - analysis: AnalysisConfig; - analyzed_fields: { - includes: string[]; - excludes: string[]; - }; - model_memory_limit: string; - max_num_threads?: number; - create_time: number; - version: string; - allow_lazy_start?: boolean; -} - export enum REFRESH_ANALYTICS_LIST_STATE { ERROR = 'error', IDLE = 'idle', @@ -379,7 +321,8 @@ export const useRefreshAnalyticsList = ( callback: { isLoading?(d: boolean): void; onRefresh?(): void; - } = {} + } = {}, + isManagementTable = false ) => { useEffect(() => { const distinct$ = refreshAnalyticsList$.pipe(distinctUntilChanged()); @@ -387,13 +330,17 @@ export const useRefreshAnalyticsList = ( const subscriptions: Subscription[] = []; if (typeof callback.onRefresh === 'function') { - // initial call to refresh - callback.onRefresh(); + // required in order to fetch the DFA jobs on the management page + if (isManagementTable) callback.onRefresh(); subscriptions.push( distinct$ .pipe(filter((state) => state === REFRESH_ANALYTICS_LIST_STATE.REFRESH)) - .subscribe(() => typeof callback.onRefresh === 'function' && callback.onRefresh()) + .subscribe(() => { + if (typeof callback.onRefresh === 'function') { + callback.onRefresh(); + } + }) ); } @@ -410,7 +357,7 @@ export const useRefreshAnalyticsList = ( return () => { subscriptions.map((sub) => sub.unsubscribe()); }; - }, []); + }, [callback.onRefresh]); return { refresh: () => { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts index 1b99aac812fcd..847aefefbc6c8 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts @@ -13,13 +13,13 @@ import { isClassificationAnalysis, isOutlierAnalysis, isRegressionAnalysis, - DataFrameAnalyticsConfig, } from './analytics'; import { Field } from '../../../../common/types/fields'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; import { newJobCapsService } from '../../services/new_job_capabilities_service'; import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants'; +import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics'; export type EsId = string; export type EsDocSource = Record; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts index eb38a23d10eef..c162cb2754c10 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts @@ -12,7 +12,8 @@ import { ml } from '../../services/ml_api_service'; import { isKeywordAndTextType } from '../common/fields'; import { SavedSearchQuery } from '../../contexts/ml'; -import { DataFrameAnalyticsConfig, INDEX_STATUS } from './analytics'; +import { INDEX_STATUS } from './analytics'; +import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics'; export const getIndexData = async ( jobConfig: DataFrameAnalyticsConfig | undefined, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts index 65531009e4436..00d735d9a866e 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts @@ -11,10 +11,7 @@ export { isOutlierAnalysis, refreshAnalyticsList$, useRefreshAnalyticsList, - DataFrameAnalyticsId, - DataFrameAnalyticsConfig, UpdateDataFrameAnalyticsConfig, - IndexName, IndexPattern, REFRESH_ANALYTICS_LIST_STATE, ANALYSIS_CONFIG_TYPE, @@ -45,3 +42,6 @@ export { getIndexData } from './get_index_data'; export { getIndexFields } from './get_index_fields'; export { useResultsViewConfig } from './use_results_view_config'; +export { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics'; +export { DataFrameAnalyticsId } from '../../../../common/types/data_frame_analytics'; +export { IndexName } from '../../../../common/types/data_frame_analytics'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx index 2f0e2ed3428c0..da5caf8e3875a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx @@ -23,10 +23,10 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { useMlContext } from '../../../contexts/ml'; import { newJobCapsService } from '../../../services/new_job_capabilities_service'; import { ml } from '../../../services/ml_api_service'; -import { DataFrameAnalyticsId } from '../../common/analytics'; import { useCreateAnalyticsForm } from '../analytics_management/hooks/use_create_analytics_form'; import { CreateAnalyticsAdvancedEditor } from './components/create_analytics_advanced_editor'; import { AdvancedStep, ConfigurationStep, CreateStep, DetailsStep } from './components'; +import { DataFrameAnalyticsId } from '../../../../../common/types/data_frame_analytics'; export enum ANALYTICS_STEPS { CONFIGURATION, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx index 0652ec5f8acb1..81494a43193dc 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx @@ -120,10 +120,13 @@ export const DataFrameAnalyticsList: FC = ({ }, [selectedIdFromUrlInitialized, analytics]); // Subscribe to the refresh observable to trigger reloading the analytics list. - useRefreshAnalyticsList({ - isLoading: setIsLoading, - onRefresh: () => getAnalytics(true), - }); + useRefreshAnalyticsList( + { + isLoading: setIsLoading, + onRefresh: () => getAnalytics(true), + }, + isManagementTable + ); const { columns, modals } = useColumns( expandedRowItemIds, @@ -271,6 +274,7 @@ export const DataFrameAnalyticsList: FC = ({ return ( <> {modals} + {analyticsStats && ( diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx new file mode 100644 index 0000000000000..bd59749517052 --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useCallback, useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiTab, EuiTabs } from '@elastic/eui'; +import { useNavigateToPath } from '../../../../../contexts/kibana'; + +interface Tab { + id: string; + name: string; + path: string; +} + +export const AnalyticsNavigationBar: FC<{ selectedTabId?: string }> = ({ selectedTabId }) => { + const navigateToPath = useNavigateToPath(); + + const tabs = useMemo( + () => [ + { + id: 'data_frame_analytics', + name: i18n.translate('xpack.ml.dataframe.jobsTabLabel', { + defaultMessage: 'Jobs', + }), + path: '/data_frame_analytics', + }, + { + id: 'models', + name: i18n.translate('xpack.ml.dataframe.modelsTabLabel', { + defaultMessage: 'Models', + }), + path: '/data_frame_analytics/models', + }, + ], + [] + ); + + const onTabClick = useCallback(async (tab: Tab) => { + await navigateToPath(tab.path); + }, []); + + return ( + + {tabs.map((tab) => { + return ( + + {tab.name} + + ); + })} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/index.ts new file mode 100644 index 0000000000000..594033f3f8d2c --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './analytics_navigation_bar'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/delete_models_modal.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/delete_models_modal.tsx new file mode 100644 index 0000000000000..83010c684473e --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/delete_models_modal.tsx @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiOverlayMask, + EuiModal, + EuiModalHeader, + EuiModalHeaderTitle, + EuiModalBody, + EuiModalFooter, + EuiButtonEmpty, + EuiButton, + EuiSpacer, + EuiCallOut, +} from '@elastic/eui'; +import { ModelItemFull } from './models_list'; + +interface DeleteModelsModalProps { + models: ModelItemFull[]; + onClose: (deletionApproved?: boolean) => void; +} + +export const DeleteModelsModal: FC = ({ models, onClose }) => { + const modelsWithPipelines = models + .filter((model) => !!model.pipelines) + .map((model) => model.model_id); + + return ( + + + + + + + + + + + + {modelsWithPipelines.length > 0 && ( + + + + )} + + + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/expanded_row.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/expanded_row.tsx new file mode 100644 index 0000000000000..7b9329fee783b --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/expanded_row.tsx @@ -0,0 +1,370 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, Fragment } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiDescriptionList, + EuiPanel, + EuiSpacer, + EuiTabbedContent, + EuiTitle, + EuiNotificationBadge, + EuiFlexGrid, + EuiFlexItem, + EuiCodeBlock, + EuiText, + EuiHorizontalRule, + EuiFlexGroup, + EuiTextColor, +} from '@elastic/eui'; +// @ts-ignore +import { formatDate } from '@elastic/eui/lib/services/format'; +import { ModelItemFull } from './models_list'; +import { TIME_FORMAT } from '../../../../../../../common/constants/time_format'; + +interface ExpandedRowProps { + item: ModelItemFull; +} + +export const ExpandedRow: FC = ({ item }) => { + const { + inference_config: inferenceConfig, + stats, + metadata, + tags, + version, + // eslint-disable-next-line @typescript-eslint/naming-convention + estimated_operations, + // eslint-disable-next-line @typescript-eslint/naming-convention + estimated_heap_memory_usage_bytes, + // eslint-disable-next-line @typescript-eslint/naming-convention + default_field_map, + // eslint-disable-next-line @typescript-eslint/naming-convention + license_level, + pipelines, + } = item; + + const details = { + tags, + version, + estimated_operations, + estimated_heap_memory_usage_bytes, + default_field_map, + license_level, + }; + + function formatToListItems(items: Record) { + return Object.entries(items) + .map(([title, value]) => { + if (title.includes('timestamp')) { + value = formatDate(value, TIME_FORMAT); + } + return { title, description: typeof value === 'object' ? JSON.stringify(value) : value }; + }) + .filter(({ description }) => { + return description !== undefined; + }); + } + + const tabs = [ + { + id: 'details', + name: ( + + ), + content: ( + <> + + + + + +
+ +
+ + + + + + + + ), + }, + ...(inferenceConfig + ? [ + { + id: 'config', + name: ( + + ), + content: ( + <> + + + + + +
+ +
+ + + + + + {metadata?.analytics_config && ( + + + +
+ +
+ + + + + + )} + + + ), + }, + ] + : []), + { + id: 'stats', + name: ( + + ), + content: ( + <> + + + {stats.inference_stats && ( + + + +
+ +
+ + + + + + )} + {stats.ingest?.total && ( + + + +
+ +
+ + + + + {stats.ingest?.pipelines && ( + <> + + +
+ +
+ + + {Object.entries(stats.ingest.pipelines).map( + ([pipelineName, { processors, ...pipelineStats }], i) => { + return ( + + + + + +
+ {i + 1}. {pipelineName} +
+ + + + + + + + + + + +
+ +
+ + + <> + {processors.map((processor) => { + const name = Object.keys(processor)[0]; + const { stats: processorStats } = processor[name]; + return ( + + + + + +
{name}
+ + + + + + + + + + + ); + })} + + + ); + } + )} + + )} + + + )} + + + ), + }, + ...(pipelines && Object.keys(pipelines).length > 0 + ? [ + { + id: 'pipelines', + name: ( + <> + {' '} + {stats.pipeline_count} + + ), + content: ( + <> + + + {Object.entries(pipelines).map(([pipelineName, { processors, description }]) => { + return ( + + + +
{pipelineName}
+ + {description && {description}} + + +
+ +
+ + + {JSON.stringify(processors, null, 2)} + + + + ); + })} + + + ), + }, + ] + : []), + ]; + + return ( + {}} + /> + ); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/index.ts new file mode 100644 index 0000000000000..7c70a25071640 --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './models_list'; + +export enum ModelsTableToConfigMapping { + id = 'model_id', + createdAt = 'create_time', + type = 'type', +} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx new file mode 100644 index 0000000000000..3104ec55c3a6d --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx @@ -0,0 +1,509 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useState, useCallback, useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + Direction, + EuiFlexGroup, + EuiFlexItem, + EuiInMemoryTable, + EuiTitle, + EuiButton, + EuiSearchBarProps, + EuiSpacer, + EuiButtonIcon, + EuiBadge, +} from '@elastic/eui'; +// @ts-ignore +import { formatDate } from '@elastic/eui/lib/services/format'; +import { EuiBasicTableColumn } from '@elastic/eui/src/components/basic_table/basic_table'; +import { EuiTableSelectionType } from '@elastic/eui/src/components/basic_table/table_types'; +import { Action } from '@elastic/eui/src/components/basic_table/action_types'; +import { StatsBar, ModelsBarStats } from '../../../../../components/stats_bar'; +import { useInferenceApiService } from '../../../../../services/ml_api_service/inference'; +import { ModelsTableToConfigMapping } from './index'; +import { TIME_FORMAT } from '../../../../../../../common/constants/time_format'; +import { DeleteModelsModal } from './delete_models_modal'; +import { useMlKibana, useNotifications } from '../../../../../contexts/kibana'; +import { ExpandedRow } from './expanded_row'; +import { getResultsUrl } from '../analytics_list/common'; +import { + ModelConfigResponse, + ModelPipelines, + TrainedModelStat, +} from '../../../../../../../common/types/inference'; +import { + REFRESH_ANALYTICS_LIST_STATE, + refreshAnalyticsList$, + useRefreshAnalyticsList, +} from '../../../../common'; + +type Stats = Omit; + +export type ModelItem = ModelConfigResponse & { + type?: string; + stats?: Stats; + pipelines?: ModelPipelines['pipelines'] | null; +}; + +export type ModelItemFull = Required; + +export const ModelsList: FC = () => { + const { + services: { + application: { navigateToUrl, capabilities }, + }, + } = useMlKibana(); + + const canDeleteDataFrameAnalytics = capabilities.ml.canDeleteDataFrameAnalytics as boolean; + + const inferenceApiService = useInferenceApiService(); + const { toasts } = useNotifications(); + + const [searchQueryText, setSearchQueryText] = useState(''); + + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(10); + const [sortField, setSortField] = useState(ModelsTableToConfigMapping.id); + const [sortDirection, setSortDirection] = useState('asc'); + + const [isLoading, setIsLoading] = useState(false); + const [items, setItems] = useState([]); + const [selectedModels, setSelectedModels] = useState([]); + + const [modelsToDelete, setModelsToDelete] = useState([]); + + const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( + {} + ); + + /** + * Fetches inference trained models. + */ + const fetchData = useCallback(async () => { + try { + const response = await inferenceApiService.getInferenceModel(undefined, { + with_pipelines: true, + size: 1000, + }); + + const newItems = []; + const expandedItemsToRefresh = []; + + for (const model of response) { + const tableItem = { + ...model, + ...(typeof model.inference_config === 'object' + ? { type: Object.keys(model.inference_config)[0] } + : {}), + }; + newItems.push(tableItem); + + if (itemIdToExpandedRowMap[model.model_id]) { + expandedItemsToRefresh.push(tableItem); + } + } + + setItems(newItems); + + if (expandedItemsToRefresh.length > 0) { + await fetchModelsStats(expandedItemsToRefresh); + + setItemIdToExpandedRowMap( + expandedItemsToRefresh.reduce((acc, item) => { + acc[item.model_id] = ; + return acc; + }, {} as Record) + ); + } + } catch (error) { + toasts.addError(new Error(error.body?.message), { + title: i18n.translate('xpack.ml.inference.modelsList.fetchFailedErrorMessage', { + defaultMessage: 'Models fetch failed', + }), + }); + } + setIsLoading(false); + refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.IDLE); + }, [itemIdToExpandedRowMap]); + + // Subscribe to the refresh observable to trigger reloading the model list. + useRefreshAnalyticsList({ + isLoading: setIsLoading, + onRefresh: fetchData, + }); + + const modelsStats: ModelsBarStats = useMemo(() => { + return { + total: { + show: true, + value: items.length, + label: i18n.translate('xpack.ml.inference.modelsList.totalAmountLabel', { + defaultMessage: 'Total inference trained models', + }), + }, + }; + }, [items]); + + /** + * Fetches models stats and update the original object + */ + const fetchModelsStats = useCallback(async (models: ModelItem[]) => { + const modelIdsToFetch = models.map((model) => model.model_id); + + try { + const { + trained_model_stats: modelsStatsResponse, + } = await inferenceApiService.getInferenceModelStats(modelIdsToFetch); + + for (const { model_id: id, ...stats } of modelsStatsResponse) { + const model = models.find((m) => m.model_id === id); + model!.stats = stats; + } + return true; + } catch (error) { + toasts.addError(new Error(error.body.message), { + title: i18n.translate('xpack.ml.inference.modelsList.fetchModelStatsErrorMessage', { + defaultMessage: 'Fetch model stats failed', + }), + }); + } + }, []); + + /** + * Unique inference types from models + */ + const inferenceTypesOptions = useMemo(() => { + const result = items.reduce((acc, item) => { + const type = item.inference_config && Object.keys(item.inference_config)[0]; + if (type) { + acc.add(type); + } + return acc; + }, new Set()); + return [...result].map((v) => ({ + value: v, + name: v, + })); + }, [items]); + + async function prepareModelsForDeletion(models: ModelItem[]) { + // Fetch model stats to check associated pipelines + if (await fetchModelsStats(models)) { + setModelsToDelete(models as ModelItemFull[]); + } else { + toasts.addDanger( + i18n.translate('xpack.ml.inference.modelsList.unableToDeleteModelsErrorMessage', { + defaultMessage: 'Unable to delete models', + }) + ); + } + } + + /** + * Deletes the models marked for deletion. + */ + async function deleteModels() { + const modelsToDeleteIds = modelsToDelete.map((model) => model.model_id); + + try { + await Promise.all( + modelsToDeleteIds.map((modelId) => inferenceApiService.deleteInferenceModel(modelId)) + ); + setItems( + items.filter( + (model) => !modelsToDelete.some((toDelete) => toDelete.model_id === model.model_id) + ) + ); + toasts.addSuccess( + i18n.translate('xpack.ml.inference.modelsList.successfullyDeletedMessage', { + defaultMessage: + '{modelsCount, plural, one {Model {modelsToDeleteIds}} other {# models}} {modelsCount, plural, one {has} other {have}} been successfully deleted', + values: { + modelsCount: modelsToDeleteIds.length, + modelsToDeleteIds: modelsToDeleteIds.join(', '), + }, + }) + ); + } catch (error) { + toasts.addError(new Error(error?.body?.message), { + title: i18n.translate('xpack.ml.inference.modelsList.fetchDeletionErrorMessage', { + defaultMessage: '{modelsCount, plural, one {Model} other {Models}} deletion failed', + values: { + modelsCount: modelsToDeleteIds.length, + }, + }), + }); + } + } + + /** + * Table actions + */ + const actions: Array> = [ + { + name: i18n.translate('xpack.ml.inference.modelsList.viewTrainingDataActionLabel', { + defaultMessage: 'View training data', + }), + description: i18n.translate('xpack.ml.inference.modelsList.viewTrainingDataActionLabel', { + defaultMessage: 'View training data', + }), + icon: 'list', + type: 'icon', + available: (item) => item.metadata?.analytics_config?.id, + onClick: async (item) => { + await navigateToUrl( + getResultsUrl( + item.metadata?.analytics_config.id, + Object.keys(item.metadata?.analytics_config.analysis)[0] + ) + ); + }, + isPrimary: true, + }, + { + name: i18n.translate('xpack.ml.inference.modelsList.deleteModelActionLabel', { + defaultMessage: 'Delete model', + }), + description: i18n.translate('xpack.ml.inference.modelsList.deleteModelActionLabel', { + defaultMessage: 'Delete model', + }), + icon: 'trash', + type: 'icon', + color: 'danger', + isPrimary: false, + onClick: async (model) => { + await prepareModelsForDeletion([model]); + }, + available: (item) => canDeleteDataFrameAnalytics, + enabled: (item) => { + // TODO check for permissions to delete ingest pipelines. + // ATM undefined means pipelines fetch failed server-side. + return !item.pipelines; + }, + }, + ]; + + const toggleDetails = async (item: ModelItem) => { + const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap }; + if (itemIdToExpandedRowMapValues[item.model_id]) { + delete itemIdToExpandedRowMapValues[item.model_id]; + } else { + await fetchModelsStats([item]); + itemIdToExpandedRowMapValues[item.model_id] = ; + } + setItemIdToExpandedRowMap(itemIdToExpandedRowMapValues); + }; + + const columns: Array> = [ + { + align: 'left', + width: '40px', + isExpander: true, + render: (item: ModelItem) => ( + + ), + }, + { + field: ModelsTableToConfigMapping.id, + name: i18n.translate('xpack.ml.inference.modelsList.modelIdHeader', { + defaultMessage: 'ID', + }), + sortable: true, + truncateText: true, + }, + { + field: ModelsTableToConfigMapping.type, + name: i18n.translate('xpack.ml.inference.modelsList.typeHeader', { + defaultMessage: 'Type', + }), + sortable: true, + align: 'left', + render: (type: string) => {type}, + }, + { + field: ModelsTableToConfigMapping.createdAt, + name: i18n.translate('xpack.ml.inference.modelsList.createdAtHeader', { + defaultMessage: 'Created at', + }), + dataType: 'date', + render: (date: string) => formatDate(date, TIME_FORMAT), + sortable: true, + }, + { + name: i18n.translate('xpack.ml.inference.modelsList.actionsHeader', { + defaultMessage: 'Actions', + }), + actions, + }, + ]; + + const pagination = { + initialPageIndex: pageIndex, + initialPageSize: pageSize, + totalItemCount: items.length, + pageSizeOptions: [10, 20, 50], + hidePerPageOptions: false, + }; + + const sorting = { + sort: { + field: sortField, + direction: sortDirection, + }, + }; + const search: EuiSearchBarProps = { + query: searchQueryText, + onChange: (searchChange) => { + if (searchChange.error !== null) { + return false; + } + setSearchQueryText(searchChange.queryText); + return true; + }, + box: { + incremental: true, + }, + ...(inferenceTypesOptions && inferenceTypesOptions.length > 0 + ? { + filters: [ + { + type: 'field_value_selection', + field: 'type', + name: i18n.translate('xpack.ml.dataframe.analyticsList.typeFilter', { + defaultMessage: 'Type', + }), + multiSelect: 'or', + options: inferenceTypesOptions, + }, + ], + } + : {}), + ...(selectedModels.length > 0 + ? { + toolsLeft: ( + + + +
+ +
+ + + + + + + + + ), + } + : {}), + }; + + const onTableChange: EuiInMemoryTable['onTableChange'] = ({ + page = { index: 0, size: 10 }, + sort = { field: ModelsTableToConfigMapping.id, direction: 'asc' }, + }) => { + const { index, size } = page; + setPageIndex(index); + setPageSize(size); + + const { field, direction } = sort; + setSortField(field); + setSortDirection(direction); + }; + + const isSelectionAllowed = canDeleteDataFrameAnalytics; + + const selection: EuiTableSelectionType | undefined = isSelectionAllowed + ? { + selectableMessage: (selectable, item) => { + return selectable + ? i18n.translate('xpack.ml.inference.modelsList.selectableMessage', { + defaultMessage: 'Select a model', + }) + : i18n.translate('xpack.ml.inference.modelsList.disableSelectableMessage', { + defaultMessage: 'Model has associated pipelines', + }); + }, + selectable: (item) => !item.pipelines, + onSelectionChange: (selectedItems) => { + setSelectedModels(selectedItems); + }, + } + : undefined; + + return ( + <> + + + {modelsStats && ( + + + + )} + + +
+ ({ + 'data-test-subj': `mlModelsTableRow row-${item.model_id}`, + })} + /> +
+ {modelsToDelete.length > 0 && ( + { + if (deletionApproved) { + await deleteModels(); + } + setModelsToDelete([]); + }} + models={modelsToDelete} + /> + )} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts index f932e4d0db7d7..4926decaa7f9c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts @@ -8,13 +8,12 @@ import { DeepPartial, DeepReadonly } from '../../../../../../../common/types/com import { checkPermission } from '../../../../../capabilities/check_capabilities'; import { mlNodesAvailable } from '../../../../../ml_nodes_check'; +import { ANALYSIS_CONFIG_TYPE, defaultSearchQuery } from '../../../../common/analytics'; +import { CloneDataFrameAnalyticsConfig } from '../../components/action_clone'; import { - DataFrameAnalyticsId, DataFrameAnalyticsConfig, - ANALYSIS_CONFIG_TYPE, - defaultSearchQuery, -} from '../../../../common/analytics'; -import { CloneDataFrameAnalyticsConfig } from '../../components/action_clone'; + DataFrameAnalyticsId, +} from '../../../../../../../common/types/data_frame_analytics'; export enum DEFAULT_MODEL_MEMORY_LIMIT { regression = '100mb', diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx index 1e83e6c7d0e03..7ffd477039e78 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, Fragment, useState } from 'react'; +import React, { FC, Fragment, useMemo, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; @@ -21,6 +21,7 @@ import { EuiTitle, } from '@elastic/eui'; +import { useLocation } from 'react-router-dom'; import { NavigationMenu } from '../../../components/navigation_menu'; import { DatePickerWrapper } from '../../../components/navigation_menu/date_picker_wrapper'; import { DataFrameAnalyticsList } from './components/analytics_list'; @@ -28,12 +29,17 @@ import { useRefreshInterval } from './components/analytics_list/use_refresh_inte import { RefreshAnalyticsListButton } from './components/refresh_analytics_list_button'; import { NodeAvailableWarning } from '../../../components/node_available_warning'; import { UpgradeWarning } from '../../../components/upgrade'; +import { AnalyticsNavigationBar } from './components/analytics_navigation_bar'; +import { ModelsList } from './components/models_management'; export const Page: FC = () => { const [blockRefresh, setBlockRefresh] = useState(false); useRefreshInterval(setBlockRefresh); + const location = useLocation(); + const selectedTabId = useMemo(() => location.pathname.split('/').pop(), [location]); + return ( @@ -45,7 +51,7 @@ export const Page: FC = () => {

  { - + + + {selectedTabId === 'data_frame_analytics' && ( + + )} + {selectedTabId === 'models' && } diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/index.ts b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/index.ts index 9b6bcc25c8c7e..c75a8240d28fb 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/index.ts +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/index.ts @@ -7,3 +7,4 @@ export * from './analytics_jobs_list'; export * from './analytics_job_exploration'; export * from './analytics_job_creation'; +export * from './models_list'; diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/models_list.tsx b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/models_list.tsx new file mode 100644 index 0000000000000..7bf7784d1b559 --- /dev/null +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/models_list.tsx @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; + +import { NavigateToPath } from '../../../contexts/kibana'; + +import { MlRoute, PageLoader, PageProps } from '../../router'; +import { useResolver } from '../../use_resolver'; +import { basicResolvers } from '../../resolvers'; +import { Page } from '../../../data_frame_analytics/pages/analytics_management'; +import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs'; + +export const modelsListRouteFactory = (navigateToPath: NavigateToPath): MlRoute => ({ + path: '/data_frame_analytics/models', + render: (props, deps) => , + breadcrumbs: [ + getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath), + getBreadcrumbWithUrlForApp('DATA_FRAME_ANALYTICS_BREADCRUMB', navigateToPath), + { + text: i18n.translate('xpack.ml.dataFrameAnalyticsBreadcrumbs.modelsListLabel', { + defaultMessage: 'Model Management', + }), + href: '', + }, + ], +}); + +const PageWrapper: FC = ({ location, deps }) => { + const { context } = useResolver('', undefined, deps.config, basicResolvers(deps)); + return ( + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/inference.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/inference.ts new file mode 100644 index 0000000000000..0206037b680bb --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/inference.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useMemo } from 'react'; +import { HttpFetchQuery } from 'kibana/public'; +import { HttpService } from '../http_service'; +import { basePath } from './index'; +import { useMlKibana } from '../../contexts/kibana'; +import { + ModelConfigResponse, + ModelPipelines, + TrainedModelStat, +} from '../../../../common/types/inference'; + +export interface InferenceQueryParams { + decompress_definition?: boolean; + from?: number; + include_model_definition?: boolean; + size?: number; + tags?: string; + // Custom kibana endpoint query params + with_pipelines?: boolean; +} + +export interface InferenceStatsQueryParams { + from?: number; + size?: number; +} + +export interface IngestStats { + count: number; + time_in_millis: number; + current: number; + failed: number; +} + +export interface InferenceStatsResponse { + count: number; + trained_model_stats: TrainedModelStat[]; +} + +/** + * Service with APIs calls to perform inference operations. + * @param httpService + */ +export function inferenceApiProvider(httpService: HttpService) { + const apiBasePath = basePath(); + + return { + /** + * Fetches configuration information for a trained inference model. + * + * @param modelId - Model ID, collection of Model IDs or Model ID pattern. + * Fetches all In case nothing is provided. + * @param params - Optional query params + */ + getInferenceModel(modelId?: string | string[], params?: InferenceQueryParams) { + let model = modelId ?? ''; + if (Array.isArray(modelId)) { + model = modelId.join(','); + } + + return httpService.http({ + path: `${apiBasePath}/inference${model && `/${model}`}`, + method: 'GET', + ...(params ? { query: params as HttpFetchQuery } : {}), + }); + }, + + /** + * Fetches usage information for trained inference models. + * + * @param modelId - Model ID, collection of Model IDs or Model ID pattern. + * Fetches all In case nothing is provided. + * @param params - Optional query params + */ + getInferenceModelStats(modelId?: string | string[], params?: InferenceStatsQueryParams) { + let model = modelId ?? '_all'; + if (Array.isArray(modelId)) { + model = modelId.join(','); + } + + return httpService.http({ + path: `${apiBasePath}/inference/${model}/_stats`, + method: 'GET', + }); + }, + + /** + * Fetches pipelines associated with provided models + * + * @param modelId - Model ID, collection of Model IDs. + */ + getInferenceModelPipelines(modelId: string | string[]) { + let model = modelId; + if (Array.isArray(modelId)) { + model = modelId.join(','); + } + + return httpService.http({ + path: `${apiBasePath}/inference/${model}/pipelines`, + method: 'GET', + }); + }, + + /** + * Deletes an existing trained inference model. + * + * @param modelId - Model ID + */ + deleteInferenceModel(modelId: string) { + return httpService.http({ + path: `${apiBasePath}/inference/${modelId}`, + method: 'DELETE', + }); + }, + }; +} + +type InferenceApiService = ReturnType; + +/** + * Hooks for accessing {@link InferenceApiService} in React components. + */ +export function useInferenceApiService(): InferenceApiService { + const { + services: { + mlServices: { httpService }, + }, + } = useMlKibana(); + return useMemo(() => inferenceApiProvider(httpService), [httpService]); +} diff --git a/x-pack/plugins/ml/server/client/error_wrapper.ts b/x-pack/plugins/ml/server/client/error_wrapper.ts index de53e4d4345a9..c635eb74e1f19 100644 --- a/x-pack/plugins/ml/server/client/error_wrapper.ts +++ b/x-pack/plugins/ml/server/client/error_wrapper.ts @@ -8,7 +8,9 @@ import { boomify, isBoom } from 'boom'; import { ResponseError, CustomHttpResponseOptions } from 'kibana/server'; export function wrapError(error: any): CustomHttpResponseOptions { - const boom = isBoom(error) ? error : boomify(error, { statusCode: error.status }); + const boom = isBoom(error) + ? error + : boomify(error, { statusCode: error.status ?? error.statusCode }); const statusCode = boom.output.statusCode; return { body: { diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/index.js b/x-pack/plugins/ml/server/models/data_frame_analytics/index.ts similarity index 86% rename from x-pack/plugins/ml/server/models/data_frame_analytics/index.js rename to x-pack/plugins/ml/server/models/data_frame_analytics/index.ts index 1d6b05e4a5759..fc18436ff5216 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/index.js +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/index.ts @@ -5,3 +5,4 @@ */ export { analyticsAuditMessagesProvider } from './analytics_audit_messages'; +export { modelsProvider } from './models_provider'; diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts new file mode 100644 index 0000000000000..b2a4ccdab43dc --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IScopedClusterClient } from 'kibana/server'; +import { PipelineDefinition } from '../../../common/types/inference'; + +export function modelsProvider(client: IScopedClusterClient) { + return { + /** + * Retrieves the map of model ids and associated pipelines. + * @param modelIds + */ + async getModelsPipelines(modelIds: string[]) { + const modelIdsMap = new Map | null>( + modelIds.map((id: string) => [id, null]) + ); + + const { body } = await client.asCurrentUser.ingest.getPipeline(); + + for (const [pipelineName, pipelineDefinition] of Object.entries(body)) { + const { processors } = pipelineDefinition as { processors: Array> }; + + for (const processor of processors) { + const id = processor.inference?.model_id; + if (modelIdsMap.has(id)) { + const obj = modelIdsMap.get(id); + if (obj === null) { + modelIdsMap.set(id, { [pipelineName]: pipelineDefinition }); + } else { + obj![pipelineName] = pipelineDefinition; + } + } + } + } + + return modelIdsMap; + }, + }; +} diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 3c3824a785032..f6d47639ef7c5 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -48,6 +48,7 @@ import { createSharedServices, SharedServices } from './shared_services'; import { getPluginPrivileges } from '../common/types/capabilities'; import { setupCapabilitiesSwitcher } from './lib/capabilities'; import { registerKibanaSettings } from './lib/register_settings'; +import { inferenceRoutes } from './routes/inference'; declare module 'kibana/server' { interface RequestHandlerContext { @@ -172,6 +173,8 @@ export class MlServerPlugin implements Plugin { + try { + const { modelId } = request.params; + const { with_pipelines: withPipelines, ...query } = request.query; + const { body } = await client.asInternalUser.ml.getTrainedModels({ + size: 1000, + ...query, + ...(modelId ? { model_id: modelId } : {}), + }); + const result = body.trained_model_configs; + try { + if (withPipelines) { + const pipelinesResponse = await modelsProvider(client).getModelsPipelines( + result.map(({ model_id: id }: { model_id: string }) => id) + ); + for (const model of result) { + model.pipelines = pipelinesResponse.get(model.model_id)!; + } + } + } catch (e) { + // the user might not have required permissions to fetch pipelines + // eslint-disable-next-line no-console + console.log(e); + } + + return response.ok({ + body: result, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup Inference + * + * @api {get} /api/ml/inference/:modelId/_stats Get stats of a trained inference model + * @apiName GetInferenceModelStats + * @apiDescription Retrieves usage information for trained inference models. + */ + router.get( + { + path: '/api/ml/inference/{modelId}/_stats', + validate: { + params: modelIdSchema, + }, + options: { + tags: ['access:ml:canGetDataFrameAnalytics'], + }, + }, + mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => { + try { + const { modelId } = request.params; + const { body } = await client.asInternalUser.ml.getTrainedModelsStats({ + ...(modelId ? { model_id: modelId } : {}), + }); + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup Inference + * + * @api {get} /api/ml/inference/:modelId/pipelines Get model pipelines + * @apiName GetModelPipelines + * @apiDescription Retrieves pipelines associated with a model + */ + router.get( + { + path: '/api/ml/inference/{modelId}/pipelines', + validate: { + params: modelIdSchema, + }, + options: { + tags: ['access:ml:canGetDataFrameAnalytics'], + }, + }, + mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => { + try { + const { modelId } = request.params; + const result = await modelsProvider(client).getModelsPipelines(modelId.split(',')); + return response.ok({ + body: [...result].map(([id, pipelines]) => ({ model_id: id, pipelines })), + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup Inference + * + * @api {delete} /api/ml/inference/:modelId Get stats of a trained inference model + * @apiName DeleteInferenceModel + * @apiDescription Deletes an existing trained inference model that is currently not referenced by an ingest pipeline. + */ + router.delete( + { + path: '/api/ml/inference/{modelId}', + validate: { + params: modelIdSchema, + }, + options: { + tags: ['access:ml:canDeleteDataFrameAnalytics'], + }, + }, + mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => { + try { + const { modelId } = request.params; + const { body } = await client.asInternalUser.ml.deleteTrainedModel({ + model_id: modelId, + }); + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); +} diff --git a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts new file mode 100644 index 0000000000000..896449be7896a --- /dev/null +++ b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; + +export const modelIdSchema = schema.object({ + /** + * Model ID + */ + modelId: schema.string(), +}); + +export const optionalModelIdSchema = schema.object({ + /** + * Model ID + */ + modelId: schema.maybe(schema.string()), +}); + +export const getInferenceQuerySchema = schema.object({ + size: schema.maybe(schema.string()), + with_pipelines: schema.maybe(schema.string()), +}); diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts index 474d6845ef3ec..cdd26b60d3be0 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts @@ -5,14 +5,14 @@ */ import expect from '@kbn/expect'; import { DataFrameAnalyticsConfig } from '../../../../plugins/ml/public/application/data_frame_analytics/common'; -import { - ClassificationAnalysis, - RegressionAnalysis, -} from '../../../../plugins/ml/public/application/data_frame_analytics/common/analytics'; import { FtrProviderContext } from '../../ftr_provider_context'; import { MlCommon } from './common'; import { MlApi } from './api'; +import { + ClassificationAnalysis, + RegressionAnalysis, +} from '../../../../plugins/ml/common/types/data_frame_analytics'; enum ANALYSIS_CONFIG_TYPE { OUTLIER_DETECTION = 'outlier_detection',