From 099dabd3785db46162913dee0124cb5ec37b3848 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 22 Nov 2021 06:58:35 -0500 Subject: [PATCH] [ML] Model management fixes and usability enhancements (#118240) (#119293) * [ML] update expanded rows on refresh * [ML] truncateText for start and state columns * remove deployment stats endpoints, replace with the trained models stats * update layout for pipelines * add edit icon * rename deployment actions * add panels for pipelines tab * change deployment stats layout * remove redundant fields from node details * render roles with badges * format nodes attrs * hide empty stats tab * enable trained models tests * add canViewMlNodes capability, hide Nodes tab for the viewer * remove unused translation * support force stop * import default ml capabilities in the security_solutions plugin * update translation string id * Revert "support force stop" This reverts commit 1b1a3b22 * fix translation string ids * remove mock id * rename test subject * remove getTrainedModelDeploymentStats leftovers * update tags for nodes_overview endpoint * update api integration tests * fix paddings * add info icons for ingest stats * update api integration tests for capabilities with spaces * expand first 3 pipelines by default * fix typo in the filename * add help_icon component * fix test assertion Co-authored-by: Dima Arnautov --- x-pack/plugins/ml/common/index.ts | 1 + .../plugins/ml/common/types/capabilities.ts | 2 + .../plugins/ml/common/types/trained_models.ts | 14 +- .../components/help_icon/help_icon.tsx | 23 + .../components/help_icon/index.tsx | 8 + .../services/ml_api_service/trained_models.ts | 16 - .../models_management/expanded_row.tsx | 307 ++------ .../models_management/models_list.tsx | 56 +- .../pipelines/expanded_row.tsx | 177 +++++ .../models_management/pipelines/index.ts | 8 + .../models_management/pipelines/pipelines.tsx | 119 ++++ .../trained_models/navigation_bar.tsx | 25 +- .../nodes_overview/allocated_models.tsx | 43 +- .../nodes_overview/expanded_row.tsx | 13 +- .../nodes_overview/nodes_list.tsx | 16 +- .../application/trained_models/page.tsx | 5 - .../capabilities/check_capabilities.test.ts | 4 +- .../ml/server/lib/ml_client/ml_client.ts | 5 - .../plugins/ml/server/lib/ml_client/types.ts | 1 - .../__mocks__/mock_deployment_response.json | 674 +++++++++--------- .../model_provider.test.ts | 21 +- .../data_frame_analytics/models_provider.ts | 25 +- x-pack/plugins/ml/server/routes/apidoc.json | 1 - .../ml/server/routes/trained_models.ts | 38 +- .../machine_learning/empty_ml_capabilities.ts | 35 +- .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - .../apis/ml/system/capabilities.ts | 4 +- .../apis/ml/system/space_capabilities.ts | 8 +- .../apps/ml/model_management/model_list.ts | 3 +- 30 files changed, 882 insertions(+), 774 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/components/help_icon/help_icon.tsx create mode 100644 x-pack/plugins/ml/public/application/components/help_icon/index.tsx create mode 100644 x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/expanded_row.tsx create mode 100644 x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/index.ts create mode 100644 x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/pipelines.tsx diff --git a/x-pack/plugins/ml/common/index.ts b/x-pack/plugins/ml/common/index.ts index ad9927a533a60..ea8ad43d6bb3b 100644 --- a/x-pack/plugins/ml/common/index.ts +++ b/x-pack/plugins/ml/common/index.ts @@ -14,3 +14,4 @@ export { composeValidators, patternValidator } from './util/validators'; export { isRuntimeMappings, isRuntimeField } from './util/runtime_field_utils'; export { extractErrorMessage } from './util/errors'; export type { RuntimeMappings } from './types/fields'; +export { getDefaultCapabilities as getDefaultMlCapabilities } from './types/capabilities'; diff --git a/x-pack/plugins/ml/common/types/capabilities.ts b/x-pack/plugins/ml/common/types/capabilities.ts index ed0f3595cb94c..36377aaa1ed3f 100644 --- a/x-pack/plugins/ml/common/types/capabilities.ts +++ b/x-pack/plugins/ml/common/types/capabilities.ts @@ -63,6 +63,8 @@ export const adminMlCapabilities = { // Alerts canCreateMlAlerts: false, canUseMlAlerts: false, + // Model management + canViewMlNodes: false, }; export type UserMlCapabilities = typeof userMlCapabilities; diff --git a/x-pack/plugins/ml/common/types/trained_models.ts b/x-pack/plugins/ml/common/types/trained_models.ts index 89b8a50846cb3..0670849f07f26 100644 --- a/x-pack/plugins/ml/common/types/trained_models.ts +++ b/x-pack/plugins/ml/common/types/trained_models.ts @@ -151,6 +151,8 @@ export interface TrainedModelDeploymentStatsResponse { routing_state: { routing_state: string }; average_inference_time_ms: number; last_access: number; + number_of_pending_requests: number; + start_time: number; }>; } @@ -161,11 +163,18 @@ export interface AllocatedModel { state: string; allocation_count: number; }; - model_id: string; + /** + * Not required for rendering in the Model stats + */ + model_id?: string; state: string; model_threads: number; model_size_bytes: number; node: { + /** + * Not required for rendering in the Nodes overview + */ + name?: string; average_inference_time_ms: number; inference_count: number; routing_state: { @@ -173,13 +182,14 @@ export interface AllocatedModel { reason?: string; }; last_access?: number; + number_of_pending_requests: number; + start_time: number; }; } export interface NodeDeploymentStatsResponse { id: string; name: string; - transport_address: string; attributes: Record; roles: string[]; allocated_models: AllocatedModel[]; diff --git a/x-pack/plugins/ml/public/application/components/help_icon/help_icon.tsx b/x-pack/plugins/ml/public/application/components/help_icon/help_icon.tsx new file mode 100644 index 0000000000000..5ab4fd4de5dd3 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/help_icon/help_icon.tsx @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, ReactNode } from 'react'; +import { EuiIcon, EuiToolTip } from '@elastic/eui'; + +export const HelpIcon: FC<{ content: ReactNode | string }> = ({ content }) => { + return ( + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/help_icon/index.tsx b/x-pack/plugins/ml/public/application/components/help_icon/index.tsx new file mode 100644 index 0000000000000..712f457da47c8 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/help_icon/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { HelpIcon } from './help_icon'; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts index c483b0a23c2d0..822c5059982e7 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts @@ -15,7 +15,6 @@ import { ModelPipelines, TrainedModelStat, NodesOverviewResponse, - TrainedModelDeploymentStatsResponse, } from '../../../../common/types/trained_models'; export interface InferenceQueryParams { @@ -122,21 +121,6 @@ export function trainedModelsApiProvider(httpService: HttpService) { }); }, - getTrainedModelDeploymentStats(modelId?: string | string[]) { - let model = modelId ?? '*'; - if (Array.isArray(modelId)) { - model = modelId.join(','); - } - - return httpService.http<{ - count: number; - deployment_stats: TrainedModelDeploymentStatsResponse[]; - }>({ - path: `${apiBasePath}/trained_models/${model}/deployment/_stats`, - method: 'GET', - }); - }, - getTrainedModelsNodesOverview() { return httpService.http({ path: `${apiBasePath}/trained_models/nodes_overview`, diff --git a/x-pack/plugins/ml/public/application/trained_models/models_management/expanded_row.tsx b/x-pack/plugins/ml/public/application/trained_models/models_management/expanded_row.tsx index 6dd7db1dbb7b6..469973a378c83 100644 --- a/x-pack/plugins/ml/public/application/trained_models/models_management/expanded_row.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/models_management/expanded_row.tsx @@ -5,53 +5,50 @@ * 2.0. */ -import React, { FC, Fragment, useEffect, useState } from 'react'; -import { omit } from 'lodash'; +import React, { FC, useEffect, useState } from 'react'; +import { omit, pick } from 'lodash'; import { EuiBadge, - EuiButtonEmpty, EuiCodeBlock, EuiDescriptionList, EuiFlexGrid, - EuiFlexGroup, EuiFlexItem, - EuiHorizontalRule, - EuiListGroup, EuiNotificationBadge, EuiPanel, EuiSpacer, EuiTabbedContent, - EuiText, - EuiTextColor, EuiTitle, } from '@elastic/eui'; -import { EuiDescriptionListProps } from '@elastic/eui/src/components/description_list/description_list'; +import type { EuiDescriptionListProps } from '@elastic/eui/src/components/description_list/description_list'; import { FormattedMessage } from '@kbn/i18n/react'; -import type { EuiListGroupItemProps } from '@elastic/eui/src/components/list_group/list_group_item'; -import { ModelItemFull } from './models_list'; -import { useMlKibana, useMlLocator } from '../../contexts/kibana'; +import type { ModelItemFull } from './models_list'; import { timeFormatter } from '../../../../common/util/date_utils'; import { isDefined } from '../../../../common/types/guards'; import { isPopulatedObject } from '../../../../common'; -import { ML_PAGES } from '../../../../common/constants/locator'; +import { ModelPipelines } from './pipelines'; +import { AllocatedModels } from '../nodes_overview/allocated_models'; +import type { AllocatedModel } from '../../../../common/types/trained_models'; interface ExpandedRowProps { item: ModelItemFull; } +const badgeFormatter = (items: string[]) => { + if (items.length === 0) return; + return ( +
+ {items.map((item) => ( + + {item} + + ))} +
+ ); +}; + const formatterDictionary: Record JSX.Element | string | undefined> = { - tags: (tags: string[]) => { - if (tags.length === 0) return; - return ( -
- {tags.map((tag) => ( - - {tag} - - ))} -
- ); - }, + tags: badgeFormatter, + roles: badgeFormatter, create_time: timeFormatter, timestamp: timeFormatter, }; @@ -89,11 +86,7 @@ export function formatToListItems( } export const ExpandedRow: FC = ({ item }) => { - const mlLocator = useMlLocator(); - - const [deploymentStatsItems, setDeploymentStats] = useState( - [] - ); + const [modelItems, setModelItems] = useState([]); const { inference_config: inferenceConfig, @@ -125,41 +118,32 @@ export const ExpandedRow: FC = ({ item }) => { license_level, }; - const { - services: { share }, - } = useMlKibana(); - useEffect( - function updateDeploymentState() { + function updateModelItems() { (async function () { - const { nodes, ...deploymentStats } = stats.deployment_stats ?? {}; - - if (!isPopulatedObject(deploymentStats)) return; + const deploymentStats = stats.deployment_stats; - const result = formatToListItems(deploymentStats)!; + if (!deploymentStats) return; - const items: EuiListGroupItemProps[] = await Promise.all( - nodes!.map(async (v) => { - const nodeObject = Object.values(v.node)[0]; - const href = await mlLocator!.getUrl({ - page: ML_PAGES.TRAINED_MODELS_NODES, - pageState: { - nodeId: nodeObject.name, - }, - }); - return { - label: nodeObject.name, - href, - }; - }) - ); - - result.push({ - title: 'nodes', - description: , + const items: AllocatedModel[] = deploymentStats.nodes.map((n) => { + const nodeName = Object.values(n.node)[0].name; + return { + ...deploymentStats, + node: { + ...pick(n, [ + 'average_inference_time_ms', + 'inference_count', + 'routing_state', + 'last_access', + 'number_of_pending_requests', + 'start_time', + ]), + name: nodeName, + } as AllocatedModel['node'], + }; }); - setDeploymentStats(result); + setModelItems(items); })(); }, [stats.deployment_stats] @@ -176,7 +160,7 @@ export const ExpandedRow: FC = ({ item }) => { ), content: ( <> - + @@ -232,7 +216,7 @@ export const ExpandedRow: FC = ({ item }) => { ), content: ( <> - + @@ -280,7 +264,7 @@ export const ExpandedRow: FC = ({ item }) => { }, ] : []), - ...(isPopulatedObject(omit(stats, 'pipeline_count')) + ...(isPopulatedObject(omit(stats, ['pipeline_count', 'ingest'])) ? [ { id: 'stats', @@ -292,57 +276,33 @@ export const ExpandedRow: FC = ({ item }) => { ), content: ( <> - - {!!deploymentStatsItems?.length ? ( - <> - - -
- -
- - - - - - - ) : null} - - {stats.inference_stats && ( - + + + + {!!modelItems?.length ? ( +
- + - )} - {stats.ingest?.total && ( + ) : null} + {stats.inference_stats ? ( - +
@@ -350,99 +310,18 @@ export const ExpandedRow: FC = ({ item }) => { - - {stats.ingest?.pipelines && ( - <> - - -
- -
- - - {Object.entries(stats.ingest.pipelines).map( - ([pipelineName, { processors, ...pipelineStats }], i) => { - return ( - - - - - -
- {i + 1}. {pipelineName} -
- - - - - - - - - - - -
- -
- - - <> - {processors.map((processor) => { - const name = Object.keys(processor)[0]; - const { stats: processorStats } = processor[name]; - return ( - - - - - -
{name}
- - - - - - - - - - - ); - })} - - - ); - } - )} - - )} - )} + ) : null} ), }, ] : []), - ...(pipelines && Object.keys(pipelines).length > 0 + ...((pipelines && Object.keys(pipelines).length > 0) || stats.ingest ? [ { id: 'pipelines', @@ -457,66 +336,8 @@ export const ExpandedRow: FC = ({ item }) => { ), content: ( <> - - - {Object.entries(pipelines).map( - ([pipelineName, { processors, description: pipelineDescription }]) => { - return ( - - - - - -
{pipelineName}
- - - - { - const locator = share.url.locators.get( - 'INGEST_PIPELINES_APP_LOCATOR' - ); - if (!locator) return; - locator.navigate({ - page: 'pipeline_edit', - pipelineId: pipelineName, - absolute: true, - }); - }} - > - - - - - - {pipelineDescription && {pipelineDescription}} - - -
- -
- - - {JSON.stringify(processors, null, 2)} - - - - ); - } - )} - + + ), }, diff --git a/x-pack/plugins/ml/public/application/trained_models/models_management/models_list.tsx b/x-pack/plugins/ml/public/application/trained_models/models_management/models_list.tsx index 9c3cc1f93a9cd..ce0e47df292de 100644 --- a/x-pack/plugins/ml/public/application/trained_models/models_management/models_list.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/models_management/models_list.tsx @@ -6,7 +6,6 @@ */ import React, { FC, useCallback, useEffect, useMemo, useState } from 'react'; -import { omit } from 'lodash'; import { EuiBadge, EuiButton, @@ -153,9 +152,7 @@ export const ModelsList: FC = () => { } // Need to fetch state for 3rd party models to enable/disable actions - await fetchAndPopulateDeploymentStats( - newItems.filter((v) => v.model_type.includes('pytorch')) - ); + await fetchModelsStats(newItems.filter((v) => v.model_type.includes('pytorch'))); setItems(newItems); @@ -237,39 +234,6 @@ export const ModelsList: FC = () => { } }, []); - /** - * Updates model items with deployment stats; - * - * We have to fetch all deployment stats on each update, - * because for stopped models the API returns 404 response. - */ - const fetchAndPopulateDeploymentStats = useCallback(async (modelItems: ModelItem[]) => { - try { - const { deployment_stats: deploymentStats } = - await trainedModelsApiService.getTrainedModelDeploymentStats('*'); - - for (const deploymentStat of deploymentStats) { - const deployedModel = modelItems.find( - (model) => model.model_id === deploymentStat.model_id - ); - - if (deployedModel) { - deployedModel.stats = { - ...(deployedModel.stats ?? {}), - deployment_stats: omit(deploymentStat, 'model_id'), - }; - } - } - } catch (error) { - displayErrorToast( - error, - i18n.translate('xpack.ml.trainedModels.modelsList.fetchDeploymentStatsErrorMessage', { - defaultMessage: 'Fetch deployment stats failed', - }) - ); - } - }, []); - /** * Unique inference types from models */ @@ -398,11 +362,11 @@ export const ModelsList: FC = () => { }, }, { - name: i18n.translate('xpack.ml.inference.modelsList.startModelAllocationActionLabel', { - defaultMessage: 'Start allocation', + name: i18n.translate('xpack.ml.inference.modelsList.startModelDeploymentActionLabel', { + defaultMessage: 'Start deployment', }), - description: i18n.translate('xpack.ml.inference.modelsList.startModelAllocationActionLabel', { - defaultMessage: 'Start allocation', + description: i18n.translate('xpack.ml.inference.modelsList.startModelDeploymentActionLabel', { + defaultMessage: 'Start deployment', }), icon: 'play', type: 'icon', @@ -442,11 +406,11 @@ export const ModelsList: FC = () => { }, }, { - name: i18n.translate('xpack.ml.inference.modelsList.stopModelAllocationActionLabel', { - defaultMessage: 'Stop allocation', + name: i18n.translate('xpack.ml.inference.modelsList.stopModelDeploymentActionLabel', { + defaultMessage: 'Stop deployment', }), - description: i18n.translate('xpack.ml.inference.modelsList.stopModelAllocationActionLabel', { - defaultMessage: 'Stop allocation', + description: i18n.translate('xpack.ml.inference.modelsList.stopModelDeploymentActionLabel', { + defaultMessage: 'Stop deployment', }), icon: 'stop', type: 'icon', @@ -567,6 +531,7 @@ export const ModelsList: FC = () => { defaultMessage: 'Type', }), sortable: true, + truncateText: true, align: 'left', render: (types: string[]) => ( @@ -587,6 +552,7 @@ export const ModelsList: FC = () => { }), sortable: (item) => item.stats?.deployment_stats?.state, align: 'left', + truncateText: true, render: (model: ModelItem) => { const state = model.stats?.deployment_stats?.state; return state ? {state} : null; diff --git a/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/expanded_row.tsx b/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/expanded_row.tsx new file mode 100644 index 0000000000000..7430d50219d3e --- /dev/null +++ b/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/expanded_row.tsx @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC } from 'react'; +import { EuiBadge, EuiInMemoryTable, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { EuiBasicTableColumn } from '@elastic/eui/src/components/basic_table/basic_table'; +import { i18n } from '@kbn/i18n'; +import { useFieldFormatter } from '../../../contexts/kibana/use_field_formatter'; +import { FIELD_FORMAT_IDS } from '../../../../../../../../src/plugins/field_formats/common'; +import { IngestStatsResponse } from './pipelines'; +import { HelpIcon } from '../../../components/help_icon'; + +interface ProcessorsStatsProps { + stats: Exclude['pipelines'][string]['processors']; +} + +type ProcessorStatsItem = ProcessorsStatsProps['stats'][number][string] & { id: string }; + +export const ProcessorsStats: FC = ({ stats }) => { + const durationFormatter = useFieldFormatter(FIELD_FORMAT_IDS.DURATION); + + const items: ProcessorStatsItem[] = stats.map((v, i) => { + const key = Object.keys(v)[0]; + return { + ...v[key], + id: `${key}_${i}`, + }; + }); + + const columns: Array> = [ + { + field: 'type', + name: i18n.translate( + 'xpack.ml.trainedModels.modelsList.pipelines.processorStats.typeHeader', + { + defaultMessage: 'Processor type', + } + ), + width: '100px', + sortable: true, + truncateText: false, + render: (type: string) => { + return {type}; + }, + 'data-test-subj': 'mlProcessorStatsType', + }, + { + field: 'stats.count', + name: ( + + + + + + + } + /> + + + ), + width: '100px', + truncateText: true, + 'data-test-subj': 'mlProcessorStatsCount', + }, + { + field: 'stats.time_in_millis', + name: ( + + + + + + + } + /> + + + ), + width: '100px', + truncateText: false, + 'data-test-subj': 'mlProcessorStatsTimePerDoc', + render: (v: number) => { + return durationFormatter(v); + }, + }, + { + field: 'stats.current', + name: ( + + + + + + + } + /> + + + ), + width: '100px', + truncateText: false, + 'data-test-subj': 'mlProcessorStatsCurrent', + }, + { + field: 'stats.failed', + name: ( + + + + + + + } + /> + + + ), + width: '100px', + 'data-test-subj': 'mlProcessorStatsFailed', + }, + ]; + + return ( + + allowNeutralSort={false} + columns={columns} + hasActions={false} + isExpandable={false} + isSelectable={false} + items={items} + itemId={'id'} + rowProps={(item) => ({ + 'data-test-subj': `mlProcessorStatsTableRow row-${item.id}`, + })} + onTableChange={() => {}} + data-test-subj={'mlProcessorStatsTable'} + /> + ); +}; diff --git a/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/index.ts b/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/index.ts new file mode 100644 index 0000000000000..791561b958164 --- /dev/null +++ b/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ModelPipelines } from './pipelines'; diff --git a/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/pipelines.tsx b/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/pipelines.tsx new file mode 100644 index 0000000000000..9b2af52eb03c8 --- /dev/null +++ b/x-pack/plugins/ml/public/application/trained_models/models_management/pipelines/pipelines.tsx @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC } from 'react'; +import { + EuiButtonEmpty, + EuiCodeBlock, + EuiFlexGrid, + EuiFlexItem, + EuiTitle, + EuiPanel, + EuiAccordion, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { useMlKibana } from '../../../contexts/kibana'; +import { ModelItem } from '../models_list'; +import { ProcessorsStats } from './expanded_row'; + +export type IngestStatsResponse = Exclude['ingest']; + +interface ModelPipelinesProps { + pipelines: Exclude; + ingestStats: IngestStatsResponse; +} + +export const ModelPipelines: FC = ({ pipelines, ingestStats }) => { + const { + services: { share }, + } = useMlKibana(); + + return ( + <> + {Object.entries(pipelines).map(([pipelineName, pipelineDefinition], i) => { + // Expand first 3 pipelines by default + const initialIsOpen = i <= 2; + return ( + <> + +
{pipelineName}
+ + } + extraAction={ + { + const locator = share.url.locators.get('INGEST_PIPELINES_APP_LOCATOR'); + if (!locator) return; + locator.navigate({ + page: 'pipeline_edit', + pipelineId: pipelineName, + absolute: true, + }); + }} + iconType={'documentEdit'} + iconSide="left" + > + + + } + paddingSize="l" + initialIsOpen={initialIsOpen} + > + + {ingestStats?.pipelines ? ( + + + +
+ +
+ + + + + + ) : null} + + + + +
+ +
+ + + {JSON.stringify(pipelineDefinition, null, 2)} + + + + + + + ); + })} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/trained_models/navigation_bar.tsx b/x-pack/plugins/ml/public/application/trained_models/navigation_bar.tsx index da8605f075c2f..ec91499bdb722 100644 --- a/x-pack/plugins/ml/public/application/trained_models/navigation_bar.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/navigation_bar.tsx @@ -9,6 +9,7 @@ import React, { FC, useCallback, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiTab, EuiTabs } from '@elastic/eui'; import { useNavigateToPath } from '../contexts/kibana'; +import { checkPermission } from '../capabilities/check_capabilities'; interface Tab { id: string; @@ -21,6 +22,8 @@ export const TrainedModelsNavigationBar: FC<{ }> = ({ selectedTabId }) => { const navigateToPath = useNavigateToPath(); + const canViewMlNodes = checkPermission('canViewMlNodes'); + const tabs = useMemo(() => { const navTabs = [ { @@ -31,17 +34,21 @@ export const TrainedModelsNavigationBar: FC<{ path: '/trained_models', testSubj: 'mlTrainedModelsTab', }, - { - id: 'nodes', - name: i18n.translate('xpack.ml.trainedModels.nodesTabLabel', { - defaultMessage: 'Nodes', - }), - path: '/trained_models/nodes', - testSubj: 'mlNodesOverviewTab', - }, + ...(canViewMlNodes + ? [ + { + id: 'nodes', + name: i18n.translate('xpack.ml.trainedModels.nodesTabLabel', { + defaultMessage: 'Nodes', + }), + path: '/trained_models/nodes', + testSubj: 'mlNodesOverviewTab', + }, + ] + : []), ]; return navTabs; - }, []); + }, [canViewMlNodes]); const onTabClick = useCallback( async (tab: Tab) => { diff --git a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/allocated_models.tsx b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/allocated_models.tsx index 2aad8183b7998..f26be61fce6f7 100644 --- a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/allocated_models.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/allocated_models.tsx @@ -17,15 +17,31 @@ import { FIELD_FORMAT_IDS } from '../../../../../../../src/plugins/field_formats interface AllocatedModelsProps { models: NodeDeploymentStatsResponse['allocated_models']; + hideColumns?: string[]; } -export const AllocatedModels: FC = ({ models }) => { +export const AllocatedModels: FC = ({ + models, + hideColumns = ['node_name'], +}) => { const bytesFormatter = useFieldFormatter(FIELD_FORMAT_IDS.BYTES); const dateFormatter = useFieldFormatter(FIELD_FORMAT_IDS.DATE); const durationFormatter = useFieldFormatter(FIELD_FORMAT_IDS.DURATION); const columns: Array> = [ { + id: 'node_name', + field: 'node.name', + name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.nodeNameHeader', { + defaultMessage: 'Node name', + }), + width: '200px', + sortable: true, + truncateText: false, + 'data-test-subj': 'mlAllocatedModelsTableNodeName', + }, + { + id: 'model_id', field: 'model_id', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelNameHeader', { defaultMessage: 'Name', @@ -84,6 +100,16 @@ export const AllocatedModels: FC = ({ models }) => { return v.node.inference_count; }, }, + { + name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelStartTimeHeader', { + defaultMessage: 'Start time', + }), + width: '200px', + 'data-test-subj': 'mlAllocatedModelsTableStartedTime', + render: (v: AllocatedModel) => { + return dateFormatter(v.node.start_time); + }, + }, { name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelLastAccessHeader', { defaultMessage: 'Last access', @@ -94,6 +120,19 @@ export const AllocatedModels: FC = ({ models }) => { return dateFormatter(v.node.last_access); }, }, + { + name: i18n.translate( + 'xpack.ml.trainedModels.nodesList.modelsList.modelNumberOfPendingRequestsHeader', + { + defaultMessage: 'Pending requests', + } + ), + width: '100px', + 'data-test-subj': 'mlAllocatedModelsTableNumberOfPendingRequests', + render: (v: AllocatedModel) => { + return v.node.number_of_pending_requests; + }, + }, { name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelRoutingStateHeader', { defaultMessage: 'Routing state', @@ -110,7 +149,7 @@ export const AllocatedModels: FC = ({ models }) => { ); }, }, - ]; + ].filter((v) => !hideColumns.includes(v.id!)); return ( diff --git a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/expanded_row.tsx b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/expanded_row.tsx index 508a5689e1c9b..ba5cdd9093210 100644 --- a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/expanded_row.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/expanded_row.tsx @@ -15,15 +15,19 @@ import { EuiTitle, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import { NodeItemWithStats } from './nodes_list'; +import { NodeItem } from './nodes_list'; import { formatToListItems } from '../models_management/expanded_row'; import { AllocatedModels } from './allocated_models'; +import { useFieldFormatter } from '../../contexts/kibana/use_field_formatter'; +import { FIELD_FORMAT_IDS } from '../../../../../../../src/plugins/field_formats/common'; interface ExpandedRowProps { - item: NodeItemWithStats; + item: NodeItem; } export const ExpandedRow: FC = ({ item }) => { + const bytesFormatter = useFieldFormatter(FIELD_FORMAT_IDS.BYTES); + const { allocated_models: allocatedModels, attributes, @@ -31,6 +35,11 @@ export const ExpandedRow: FC = ({ item }) => { ...details } = item; + // Process node attributes + attributes['ml.machine_memory'] = bytesFormatter(attributes['ml.machine_memory']); + attributes['ml.max_jvm_size'] = bytesFormatter(attributes['ml.max_jvm_size']); + delete attributes['xpack.installed']; + return ( <> diff --git a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/nodes_list.tsx b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/nodes_list.tsx index b1cc18e698c9d..87211fedaea43 100644 --- a/x-pack/plugins/ml/public/application/trained_models/nodes_overview/nodes_list.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/nodes_overview/nodes_list.tsx @@ -36,10 +36,6 @@ import { useRefresh } from '../../routing/use_refresh'; export type NodeItem = NodeDeploymentStatsResponse; -export interface NodeItemWithStats extends NodeItem { - stats: any; -} - export const getDefaultNodesListState = (): ListingPageUrlState => ({ pageIndex: 0, pageSize: 10, @@ -70,6 +66,14 @@ export const NodesList: FC = () => { try { const nodesResponse = await trainedModelsApiService.getTrainedModelsNodesOverview(); setItems(nodesResponse.nodes); + + // Update expanded rows. + nodesResponse.nodes.forEach((node) => { + if (itemIdToExpandedRowMap[node.id]) { + itemIdToExpandedRowMap[node.id] = ; + } + }); + setIsLoading(false); refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.IDLE); } catch (e) { @@ -80,14 +84,14 @@ export const NodesList: FC = () => { }) ); } - }, []); + }, [itemIdToExpandedRowMap]); const toggleDetails = (item: NodeItem) => { const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap }; if (itemIdToExpandedRowMapValues[item.id]) { delete itemIdToExpandedRowMapValues[item.id]; } else { - itemIdToExpandedRowMapValues[item.id] = ; + itemIdToExpandedRowMapValues[item.id] = ; } setItemIdToExpandedRowMap(itemIdToExpandedRowMapValues); }; diff --git a/x-pack/plugins/ml/public/application/trained_models/page.tsx b/x-pack/plugins/ml/public/application/trained_models/page.tsx index 54849f3e651df..afbebf58937b3 100644 --- a/x-pack/plugins/ml/public/application/trained_models/page.tsx +++ b/x-pack/plugins/ml/public/application/trained_models/page.tsx @@ -28,14 +28,9 @@ import { ModelsList } from './models_management'; import { TrainedModelsNavigationBar } from './navigation_bar'; import { RefreshAnalyticsListButton } from '../data_frame_analytics/pages/analytics_management/components/refresh_analytics_list_button'; import { DatePickerWrapper } from '../components/navigation_menu/date_picker_wrapper'; -import { useRefreshAnalyticsList } from '../data_frame_analytics/common'; -import { useRefreshInterval } from '../data_frame_analytics/pages/analytics_management/components/analytics_list/use_refresh_interval'; import { NodesList } from './nodes_overview'; export const Page: FC = () => { - useRefreshInterval(() => {}); - - useRefreshAnalyticsList({ isLoading: () => {} }); const location = useLocation(); const selectedTabId = useMemo(() => location.pathname.split('/').pop(), [location]); diff --git a/x-pack/plugins/ml/server/lib/capabilities/check_capabilities.test.ts b/x-pack/plugins/ml/server/lib/capabilities/check_capabilities.test.ts index 920e1f703422d..c72b4d5cb5dd7 100644 --- a/x-pack/plugins/ml/server/lib/capabilities/check_capabilities.test.ts +++ b/x-pack/plugins/ml/server/lib/capabilities/check_capabilities.test.ts @@ -51,7 +51,7 @@ describe('check_capabilities', () => { ); const { capabilities } = await getCapabilities(); const count = Object.keys(capabilities).length; - expect(count).toBe(31); + expect(count).toBe(32); }); }); @@ -101,6 +101,7 @@ describe('check_capabilities', () => { expect(capabilities.canCreateDataFrameAnalytics).toBe(false); expect(capabilities.canStartStopDataFrameAnalytics).toBe(false); expect(capabilities.canCreateMlAlerts).toBe(false); + expect(capabilities.canViewMlNodes).toBe(false); }); test('full capabilities', async () => { @@ -146,6 +147,7 @@ describe('check_capabilities', () => { expect(capabilities.canDeleteDataFrameAnalytics).toBe(true); expect(capabilities.canCreateDataFrameAnalytics).toBe(true); expect(capabilities.canStartStopDataFrameAnalytics).toBe(true); + expect(capabilities.canViewMlNodes).toBe(true); }); test('upgrade in progress with full capabilities', async () => { diff --git a/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts b/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts index 6169d9ee9db47..c2b98ab1b0c29 100644 --- a/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts +++ b/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts @@ -380,11 +380,6 @@ export function getMlClient( async getTrainedModelsStats(...p: Parameters) { return mlClient.getTrainedModelsStats(...p); }, - async getTrainedModelDeploymentStats( - ...p: Parameters - ) { - return mlClient.getTrainedModelDeploymentStats(...p); - }, async startTrainedModelDeployment(...p: Parameters) { return mlClient.startTrainedModelDeployment(...p); }, diff --git a/x-pack/plugins/ml/server/lib/ml_client/types.ts b/x-pack/plugins/ml/server/lib/ml_client/types.ts index d8c65c4f56814..b4778f4e6d5b1 100644 --- a/x-pack/plugins/ml/server/lib/ml_client/types.ts +++ b/x-pack/plugins/ml/server/lib/ml_client/types.ts @@ -48,7 +48,6 @@ export type MlClientParams = | Parameters | Parameters | Parameters - | Parameters | Parameters | Parameters | Parameters diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/__mocks__/mock_deployment_response.json b/x-pack/plugins/ml/server/models/data_frame_analytics/__mocks__/mock_deployment_response.json index 0742c249b67b0..5d80fa26b4c34 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/__mocks__/mock_deployment_response.json +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/__mocks__/mock_deployment_response.json @@ -1,357 +1,355 @@ -{ - "count" : 4, - "deployment_stats" : [ - { - "model_id" : "distilbert-base-uncased-finetuned-sst-2-english", - "model_size_bytes" : 267386880, - "inference_threads" : 1, - "model_threads" : 1, - "state" : "started", - "allocation_status" : { - "allocation_count" : 2, - "target_allocation_count" : 3, - "state" : "started" - }, - "nodes" : [ - { - "node" : { - "3qIoLFnbSi-DwVrYioUCdw" : { - "name" : "node3", - "ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg", - "transport_address" : "10.142.0.2:9353", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "ingest", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 +[ + { + "model_id": "distilbert-base-uncased-finetuned-sst-2-english", + "model_size_bytes": 267386880, + "inference_threads": 1, + "model_threads": 1, + "state": "started", + "allocation_status": { + "allocation_count": 2, + "target_allocation_count": 3, + "state": "started" + }, + "nodes": [ + { + "node": { + "3qIoLFnbSi-DwVrYioUCdw": { + "name": "node3", + "ephemeral_id": "WeA49KLuRPmJM_ulLx0ANg", + "transport_address": "10.142.0.2:9353", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "ingest", + "master", + "ml", + "transform" + ] + } }, - { - "node" : { - "DpCy7SOBQla3pu0Dq-tnYw" : { - "name" : "node2", - "ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g", - "transport_address" : "10.142.0.2:9352", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "failed", - "reason" : "The object cannot be set twice!" + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + }, + { + "node": { + "DpCy7SOBQla3pu0Dq-tnYw": { + "name": "node2", + "ephemeral_id": "17qcsXsNTYqbJ6uwSvdl9g", + "transport_address": "10.142.0.2:9352", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml", + "transform" + ] } }, - { - "node" : { - "pt7s6lKHQJaP4QHKtU-Q0Q" : { - "name" : "node1", - "ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q", - "transport_address" : "10.142.0.2:9351", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 + "routing_state": { + "routing_state": "failed", + "reason": "The object cannot be set twice!" } - ] - }, - { - "model_id" : "elastic__distilbert-base-cased-finetuned-conll03-english", - "model_size_bytes" : 260947500, - "inference_threads" : 1, - "model_threads" : 1, - "state" : "started", - "allocation_status" : { - "allocation_count" : 2, - "target_allocation_count" : 3, - "state" : "started" }, - "nodes" : [ - { - "node" : { - "3qIoLFnbSi-DwVrYioUCdw" : { - "name" : "node3", - "ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg", - "transport_address" : "10.142.0.2:9353", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "ingest", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 + { + "node": { + "pt7s6lKHQJaP4QHKtU-Q0Q": { + "name": "node1", + "ephemeral_id": "nMJBE9WSRQSWotk0zDPi_Q", + "transport_address": "10.142.0.2:9351", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml" + ] + } }, - { - "node" : { - "DpCy7SOBQla3pu0Dq-tnYw" : { - "name" : "node2", - "ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g", - "transport_address" : "10.142.0.2:9352", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "failed", - "reason" : "The object cannot be set twice!" + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + } + ] + }, + { + "model_id": "elastic__distilbert-base-cased-finetuned-conll03-english", + "model_size_bytes": 260947500, + "inference_threads": 1, + "model_threads": 1, + "state": "started", + "allocation_status": { + "allocation_count": 2, + "target_allocation_count": 3, + "state": "started" + }, + "nodes": [ + { + "node": { + "3qIoLFnbSi-DwVrYioUCdw": { + "name": "node3", + "ephemeral_id": "WeA49KLuRPmJM_ulLx0ANg", + "transport_address": "10.142.0.2:9353", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "ingest", + "master", + "ml", + "transform" + ] } }, - { - "node" : { - "pt7s6lKHQJaP4QHKtU-Q0Q" : { - "name" : "node1", - "ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q", - "transport_address" : "10.142.0.2:9351", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + }, + { + "node": { + "DpCy7SOBQla3pu0Dq-tnYw": { + "name": "node2", + "ephemeral_id": "17qcsXsNTYqbJ6uwSvdl9g", + "transport_address": "10.142.0.2:9352", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml", + "transform" + ] + } + }, + "routing_state": { + "routing_state": "failed", + "reason": "The object cannot be set twice!" } - ] - }, - { - "model_id" : "sentence-transformers__msmarco-minilm-l-12-v3", - "model_size_bytes" : 133378867, - "inference_threads" : 1, - "model_threads" : 1, - "state" : "started", - "allocation_status" : { - "allocation_count" : 2, - "target_allocation_count" : 3, - "state" : "started" }, - "nodes" : [ - { - "node" : { - "3qIoLFnbSi-DwVrYioUCdw" : { - "name" : "node3", - "ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg", - "transport_address" : "10.142.0.2:9353", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "ingest", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 + { + "node": { + "pt7s6lKHQJaP4QHKtU-Q0Q": { + "name": "node1", + "ephemeral_id": "nMJBE9WSRQSWotk0zDPi_Q", + "transport_address": "10.142.0.2:9351", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml" + ] + } }, - { - "node" : { - "DpCy7SOBQla3pu0Dq-tnYw" : { - "name" : "node2", - "ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g", - "transport_address" : "10.142.0.2:9352", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "failed", - "reason" : "The object cannot be set twice!" + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + } + ] + }, + { + "model_id": "sentence-transformers__msmarco-minilm-l-12-v3", + "model_size_bytes": 133378867, + "inference_threads": 1, + "model_threads": 1, + "state": "started", + "allocation_status": { + "allocation_count": 2, + "target_allocation_count": 3, + "state": "started" + }, + "nodes": [ + { + "node": { + "3qIoLFnbSi-DwVrYioUCdw": { + "name": "node3", + "ephemeral_id": "WeA49KLuRPmJM_ulLx0ANg", + "transport_address": "10.142.0.2:9353", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "ingest", + "master", + "ml", + "transform" + ] } }, - { - "node" : { - "pt7s6lKHQJaP4QHKtU-Q0Q" : { - "name" : "node1", - "ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q", - "transport_address" : "10.142.0.2:9351", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + }, + { + "node": { + "DpCy7SOBQla3pu0Dq-tnYw": { + "name": "node2", + "ephemeral_id": "17qcsXsNTYqbJ6uwSvdl9g", + "transport_address": "10.142.0.2:9352", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml", + "transform" + ] + } + }, + "routing_state": { + "routing_state": "failed", + "reason": "The object cannot be set twice!" } - ] - }, - { - "model_id" : "typeform__mobilebert-uncased-mnli", - "model_size_bytes" : 100139008, - "inference_threads" : 1, - "model_threads" : 1, - "state" : "started", - "allocation_status" : { - "allocation_count" : 2, - "target_allocation_count" : 3, - "state" : "started" }, - "nodes" : [ - { - "node" : { - "3qIoLFnbSi-DwVrYioUCdw" : { - "name" : "node3", - "ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg", - "transport_address" : "10.142.0.2:9353", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "ingest", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 + { + "node": { + "pt7s6lKHQJaP4QHKtU-Q0Q": { + "name": "node1", + "ephemeral_id": "nMJBE9WSRQSWotk0zDPi_Q", + "transport_address": "10.142.0.2:9351", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml" + ] + } + }, + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + } + ] + }, + { + "model_id": "typeform__mobilebert-uncased-mnli", + "model_size_bytes": 100139008, + "inference_threads": 1, + "model_threads": 1, + "state": "started", + "allocation_status": { + "allocation_count": 2, + "target_allocation_count": 3, + "state": "started" + }, + "nodes": [ + { + "node": { + "3qIoLFnbSi-DwVrYioUCdw": { + "name": "node3", + "ephemeral_id": "WeA49KLuRPmJM_ulLx0ANg", + "transport_address": "10.142.0.2:9353", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "ingest", + "master", + "ml", + "transform" + ] + } }, - { - "node" : { - "DpCy7SOBQla3pu0Dq-tnYw" : { - "name" : "node2", - "ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g", - "transport_address" : "10.142.0.2:9352", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml", - "transform" - ] - } - }, - "routing_state" : { - "routing_state" : "failed", - "reason" : "The object cannot be set twice!" + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + }, + { + "node": { + "DpCy7SOBQla3pu0Dq-tnYw": { + "name": "node2", + "ephemeral_id": "17qcsXsNTYqbJ6uwSvdl9g", + "transport_address": "10.142.0.2:9352", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml", + "transform" + ] } }, - { - "node" : { - "pt7s6lKHQJaP4QHKtU-Q0Q" : { - "name" : "node1", - "ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q", - "transport_address" : "10.142.0.2:9351", - "attributes" : { - "ml.machine_memory" : "15599742976", - "xpack.installed" : "true", - "ml.max_jvm_size" : "1073741824" - }, - "roles" : [ - "data", - "master", - "ml" - ] - } - }, - "routing_state" : { - "routing_state" : "started" - }, - "inference_count" : 0, - "average_inference_time_ms" : 0.0 + "routing_state": { + "routing_state": "failed", + "reason": "The object cannot be set twice!" } - ] - } - ] -} + }, + { + "node": { + "pt7s6lKHQJaP4QHKtU-Q0Q": { + "name": "node1", + "ephemeral_id": "nMJBE9WSRQSWotk0zDPi_Q", + "transport_address": "10.142.0.2:9351", + "attributes": { + "ml.machine_memory": "15599742976", + "xpack.installed": "true", + "ml.max_jvm_size": "1073741824" + }, + "roles": [ + "data", + "master", + "ml" + ] + } + }, + "routing_state": { + "routing_state": "started" + }, + "inference_count": 0, + "average_inference_time_ms": 0.0 + } + ] + } +] + diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/model_provider.test.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/model_provider.test.ts index 4f5e1ee9b230c..c0d70aa471992 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/model_provider.test.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/model_provider.test.ts @@ -104,8 +104,16 @@ describe('Model service', () => { }, } as unknown as jest.Mocked; const mlClient = { - getTrainedModelDeploymentStats: jest.fn(() => { - return Promise.resolve({ body: mockResponse }); + getTrainedModelsStats: jest.fn(() => { + return Promise.resolve({ + body: { + trained_model_stats: mockResponse.map((v) => { + return { + deployment_stats: v, + }; + }), + }, + }); }), } as unknown as jest.Mocked; const memoryOverviewService = { @@ -214,9 +222,7 @@ describe('Model service', () => { 'ml.max_jvm_size': '1073741824', 'xpack.installed': 'true', }, - host: '10.10.10.2', id: '3qIoLFnbSi-DwVrYioUCdw', - ip: '10.10.10.2:9353', memory_overview: { anomaly_detection: { total: 0, @@ -251,7 +257,6 @@ describe('Model service', () => { }, }, roles: ['data', 'ingest', 'master', 'ml', 'transform'], - transport_address: '10.10.10.2:9353', }, { name: 'node2', @@ -334,9 +339,7 @@ describe('Model service', () => { 'ml.max_jvm_size': '1073741824', 'xpack.installed': 'true', }, - host: '10.10.10.2', id: 'DpCy7SOBQla3pu0Dq-tnYw', - ip: '10.10.10.2:9352', memory_overview: { anomaly_detection: { total: 0, @@ -371,7 +374,6 @@ describe('Model service', () => { }, }, roles: ['data', 'master', 'ml', 'transform'], - transport_address: '10.10.10.2:9352', }, { allocated_models: [ @@ -457,9 +459,7 @@ describe('Model service', () => { 'ml.max_jvm_size': '1073741824', 'xpack.installed': 'true', }, - host: '10.10.10.2', id: 'pt7s6lKHQJaP4QHKtU-Q0Q', - ip: '10.10.10.2:9351', memory_overview: { anomaly_detection: { total: 0, @@ -495,7 +495,6 @@ describe('Model service', () => { }, name: 'node1', roles: ['data', 'master', 'ml'], - transport_address: '10.10.10.2:9351', }, ], }); diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts index 2f40081f1458d..104e320e7fab1 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts @@ -19,18 +19,11 @@ import { NATIVE_EXECUTABLE_CODE_OVERHEAD, } from '../memory_overview/memory_overview_service'; import { TrainedModelDeploymentStatsResponse } from '../../../common/types/trained_models'; +import { isDefined } from '../../../common/types/guards'; export type ModelService = ReturnType; -const NODE_FIELDS = [ - 'attributes', - 'name', - 'roles', - 'ip', - 'host', - 'transport_address', - 'version', -] as const; +const NODE_FIELDS = ['attributes', 'name', 'roles', 'version'] as const; export type RequiredNodeFields = Pick; @@ -87,8 +80,11 @@ export function modelsProvider( throw new Error('Memory overview service is not provided'); } - const { body: deploymentStats } = await mlClient.getTrainedModelDeploymentStats({ - model_id: '*', + const { + body: { trained_model_stats: trainedModelStats }, + } = await mlClient.getTrainedModelsStats({ + model_id: '_all', + size: 10000, }); const { @@ -105,7 +101,12 @@ export function modelsProvider( const nodeFields = pick(node, NODE_FIELDS) as RequiredNodeFields; const allocatedModels = ( - deploymentStats.deployment_stats as TrainedModelDeploymentStatsResponse[] + trainedModelStats + .map((v) => { + // @ts-ignore new prop + return v.deployment_stats; + }) + .filter(isDefined) as TrainedModelDeploymentStatsResponse[] ) .filter((v) => v.nodes.some((n) => Object.keys(n.node)[0] === nodeId)) .map(({ nodes, ...rest }) => { diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index b7bd92c913891..e1a839b21f7b0 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -160,7 +160,6 @@ "TrainedModels", "GetTrainedModel", "GetTrainedModelStats", - "GetTrainedModelDeploymentStats", "GetTrainedModelsNodesOverview", "GetTrainedModelPipelines", "StartTrainedModelDeployment", diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts index 1837f9e88edf3..e7696861153ff 100644 --- a/x-pack/plugins/ml/server/routes/trained_models.ts +++ b/x-pack/plugins/ml/server/routes/trained_models.ts @@ -198,7 +198,11 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization) path: '/api/ml/trained_models/nodes_overview', validate: {}, options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + tags: [ + 'access:ml:canViewMlNodes', + 'access:ml:canGetDataFrameAnalytics', + 'access:ml:canGetJobs', + ], }, }, routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response }) => { @@ -281,36 +285,4 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization) } }) ); - - /** - * @apiGroup TrainedModels - * - * @api {get} /api/ml/trained_models/:modelId/deployment/_stats Get trained model deployment stats - * @apiName GetTrainedModelDeploymentStats - * @apiDescription Gets trained model deployment stats. - */ - router.get( - { - path: '/api/ml/trained_models/{modelId}/deployment/_stats', - validate: { - params: modelIdSchema, - }, - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], - }, - }, - routeGuard.fullLicenseAPIGuard(async ({ mlClient, request, response }) => { - try { - const { modelId } = request.params; - const { body } = await mlClient.getTrainedModelDeploymentStats({ - model_id: modelId, - }); - return response.ok({ - body, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - }) - ); } diff --git a/x-pack/plugins/security_solution/common/machine_learning/empty_ml_capabilities.ts b/x-pack/plugins/security_solution/common/machine_learning/empty_ml_capabilities.ts index 772c16fc9cb99..79a0084f91923 100644 --- a/x-pack/plugins/security_solution/common/machine_learning/empty_ml_capabilities.ts +++ b/x-pack/plugins/security_solution/common/machine_learning/empty_ml_capabilities.ts @@ -6,41 +6,10 @@ */ import { MlCapabilitiesResponse } from '../../../ml/common/types/capabilities'; +import { getDefaultMlCapabilities } from '../../../ml/common'; export const emptyMlCapabilities: MlCapabilitiesResponse = { - capabilities: { - canAccessML: false, - canGetAnnotations: false, - canCreateAnnotation: false, - canDeleteAnnotation: false, - canGetJobs: false, - canCreateJob: false, - canDeleteJob: false, - canOpenJob: false, - canCloseJob: false, - canResetJob: false, - canForecastJob: false, - canGetDatafeeds: false, - canStartStopDatafeed: false, - canUpdateJob: false, - canUpdateDatafeed: false, - canPreviewDatafeed: false, - canGetCalendars: false, - canCreateCalendar: false, - canDeleteCalendar: false, - canGetFilters: false, - canCreateFilter: false, - canDeleteFilter: false, - canFindFileStructure: false, - canCreateDatafeed: false, - canDeleteDatafeed: false, - canGetDataFrameAnalytics: false, - canDeleteDataFrameAnalytics: false, - canCreateDataFrameAnalytics: false, - canStartStopDataFrameAnalytics: false, - canCreateMlAlerts: false, - canUseMlAlerts: false, - }, + capabilities: getDefaultMlCapabilities(), isPlatinumOrTrialLicense: false, mlFeatureEnabledInSpace: false, upgradeInProgress: false, diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 3b43ecf0dc16f..cd070524bb539 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -17835,8 +17835,6 @@ "xpack.ml.trainedModels.modelsList.deleteModelsButtonLabel": "削除", "xpack.ml.trainedModels.modelsList.disableSelectableMessage": "モデルにはパイプラインが関連付けられています", "xpack.ml.trainedModels.modelsList.expandedRow.analyticsConfigTitle": "分析構成", - "xpack.ml.trainedModels.modelsList.expandedRow.byPipelineTitle": "パイプライン別", - "xpack.ml.trainedModels.modelsList.expandedRow.byProcessorTitle": "プロセッサー別", "xpack.ml.trainedModels.modelsList.expandedRow.configTabLabel": "構成", "xpack.ml.trainedModels.modelsList.expandedRow.detailsTabLabel": "詳細", "xpack.ml.trainedModels.modelsList.expandedRow.detailsTitle": "詳細", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 3ac2787a72d08..ed5f2c897ab35 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -18110,8 +18110,6 @@ "xpack.ml.trainedModels.modelsList.deleteModelsButtonLabel": "删除", "xpack.ml.trainedModels.modelsList.disableSelectableMessage": "模型有关联的管道", "xpack.ml.trainedModels.modelsList.expandedRow.analyticsConfigTitle": "分析配置", - "xpack.ml.trainedModels.modelsList.expandedRow.byPipelineTitle": "按管道", - "xpack.ml.trainedModels.modelsList.expandedRow.byProcessorTitle": "按处理器", "xpack.ml.trainedModels.modelsList.expandedRow.configTabLabel": "配置", "xpack.ml.trainedModels.modelsList.expandedRow.detailsTabLabel": "详情", "xpack.ml.trainedModels.modelsList.expandedRow.detailsTitle": "详情", diff --git a/x-pack/test/api_integration/apis/ml/system/capabilities.ts b/x-pack/test/api_integration/apis/ml/system/capabilities.ts index 4eb040d031c2e..d0df53dfee343 100644 --- a/x-pack/test/api_integration/apis/ml/system/capabilities.ts +++ b/x-pack/test/api_integration/apis/ml/system/capabilities.ts @@ -45,7 +45,7 @@ export default ({ getService }: FtrProviderContext) => { it('should have the right number of capabilities', async () => { const { capabilities } = await runRequest(USER.ML_POWERUSER); - expect(Object.keys(capabilities).length).to.eql(31); + expect(Object.keys(capabilities).length).to.eql(32); }); it('should get viewer capabilities', async () => { @@ -83,6 +83,7 @@ export default ({ getService }: FtrProviderContext) => { canGetAnnotations: true, canCreateAnnotation: true, canDeleteAnnotation: true, + canViewMlNodes: false, }); }); @@ -121,6 +122,7 @@ export default ({ getService }: FtrProviderContext) => { canGetAnnotations: true, canCreateAnnotation: true, canDeleteAnnotation: true, + canViewMlNodes: true, }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/system/space_capabilities.ts b/x-pack/test/api_integration/apis/ml/system/space_capabilities.ts index 6d6a00e882689..b51b87457caa2 100644 --- a/x-pack/test/api_integration/apis/ml/system/space_capabilities.ts +++ b/x-pack/test/api_integration/apis/ml/system/space_capabilities.ts @@ -71,11 +71,11 @@ export default ({ getService }: FtrProviderContext) => { it('should have the right number of capabilities - space with ML', async () => { const { capabilities } = await runRequest(USER.ML_POWERUSER, idSpaceWithMl); - expect(Object.keys(capabilities).length).to.eql(31); + expect(Object.keys(capabilities).length).to.eql(32); }); it('should have the right number of capabilities - space without ML', async () => { const { capabilities } = await runRequest(USER.ML_POWERUSER, idSpaceNoMl); - expect(Object.keys(capabilities).length).to.eql(31); + expect(Object.keys(capabilities).length).to.eql(32); }); it('should get viewer capabilities - space with ML', async () => { @@ -112,6 +112,7 @@ export default ({ getService }: FtrProviderContext) => { canGetAnnotations: true, canCreateAnnotation: true, canDeleteAnnotation: true, + canViewMlNodes: false, }); }); @@ -149,6 +150,7 @@ export default ({ getService }: FtrProviderContext) => { canGetAnnotations: false, canCreateAnnotation: false, canDeleteAnnotation: false, + canViewMlNodes: false, }); }); @@ -186,6 +188,7 @@ export default ({ getService }: FtrProviderContext) => { canGetAnnotations: true, canCreateAnnotation: true, canDeleteAnnotation: true, + canViewMlNodes: true, }); }); @@ -223,6 +226,7 @@ export default ({ getService }: FtrProviderContext) => { canGetAnnotations: false, canCreateAnnotation: false, canDeleteAnnotation: false, + canViewMlNodes: false, }); }); }); diff --git a/x-pack/test/functional/apps/ml/model_management/model_list.ts b/x-pack/test/functional/apps/ml/model_management/model_list.ts index aac1ad5b1e50b..955639dbe60a4 100644 --- a/x-pack/test/functional/apps/ml/model_management/model_list.ts +++ b/x-pack/test/functional/apps/ml/model_management/model_list.ts @@ -10,8 +10,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const ml = getService('ml'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/118251 - describe.skip('trained models', function () { + describe('trained models', function () { before(async () => { await ml.trainedModels.createTestTrainedModels('classification', 15, true); await ml.trainedModels.createTestTrainedModels('regression', 15);