From 327448b726a916d42fa6138578b0edce0637834c Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Tue, 8 Aug 2023 14:01:28 -0400 Subject: [PATCH 1/5] [ML] Adds ability to deploy trained models for data frame analytics jobs (#162537) ## Summary Related issue: https://github.com/elastic/kibana/issues/161026 This PR adds a 'Deploy model' action in Machine Learning > Model Management > Trained models. Action in list: image Details step: image Configure processor: image Configure processor edit: image Configure processor additional: image Handle failures: image image image Test: image Create: image Create success: image ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- packages/kbn-doc-links/src/get_doc_links.ts | 1 + packages/kbn-doc-links/src/types.ts | 1 + .../add_inference_pipeline_flyout.tsx | 186 +++++++++ .../add_inference_pipeline_footer.tsx | 96 +++++ ...dd_inference_pipeline_horizontal_steps.tsx | 118 ++++++ .../additional_advanced_settings.tsx | 162 ++++++++ .../components/on_failure_configuration.tsx | 270 ++++++++++++ .../components/pipeline_details.tsx | 223 ++++++++++ .../components/processor_configuration.tsx | 391 ++++++++++++++++++ .../components/review_and_create_pipeline.tsx | 214 ++++++++++ .../components/save_changes_button.tsx | 24 ++ .../ml_inference/components/test_pipeline.tsx | 279 +++++++++++++ .../components/ml_inference/constants.ts | 61 +++ .../ml_inference/get_pipeline_config.ts | 41 ++ .../components/ml_inference/get_steps.ts | 47 +++ .../ml_inference/hooks/use_fetch_pipelines.ts | 47 +++ .../components/ml_inference/index.ts | 8 + .../components/ml_inference/state.ts | 83 ++++ .../components/ml_inference/types.ts | 48 +++ .../components/ml_inference/validation.ts | 118 ++++++ .../model_management/model_actions.tsx | 51 +++ .../model_management/models_list.tsx | 9 + .../services/ml_api_service/trained_models.ts | 28 +- .../apidoc_scripts/apidoc_config/apidoc.json | 2 + .../model_management/models_provider.ts | 64 +++ .../server/routes/schemas/inference_schema.ts | 10 + .../ml/server/routes/trained_models.ts | 73 ++++ 27 files changed, 2651 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/add_inference_pipeline_flyout.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_footer.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_horizontal_steps.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/additional_advanced_settings.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/on_failure_configuration.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/pipeline_details.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/processor_configuration.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/review_and_create_pipeline.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/save_changes_button.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/components/test_pipeline.tsx create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/constants.ts create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/get_pipeline_config.ts create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/get_steps.ts create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/hooks/use_fetch_pipelines.ts create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/index.ts create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/state.ts create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/types.ts create mode 100644 x-pack/plugins/ml/public/application/components/ml_inference/validation.ts diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts index 249820124ef7a..7bf1b32195dd5 100644 --- a/packages/kbn-doc-links/src/get_doc_links.ts +++ b/packages/kbn-doc-links/src/get_doc_links.ts @@ -319,6 +319,7 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => { overview: `${KIBANA_DOCS}upgrade-assistant.html`, batchReindex: `${KIBANA_DOCS}batch-start-resume-reindex.html`, remoteReindex: `${ELASTICSEARCH_DOCS}docs-reindex.html#reindex-from-remote`, + reindexWithPipeline: `${ELASTICSEARCH_DOCS}docs-reindex.html#reindex-with-an-ingest-pipeline`, }, rollupJobs: `${KIBANA_DOCS}data-rollups.html`, elasticsearch: { diff --git a/packages/kbn-doc-links/src/types.ts b/packages/kbn-doc-links/src/types.ts index dff8af667ccba..5869005187db6 100644 --- a/packages/kbn-doc-links/src/types.ts +++ b/packages/kbn-doc-links/src/types.ts @@ -298,6 +298,7 @@ export interface DocLinks { readonly overview: string; readonly batchReindex: string; readonly remoteReindex: string; + readonly reindexWithPipeline: string; }; readonly rollupJobs: string; readonly elasticsearch: Record; diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/add_inference_pipeline_flyout.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/add_inference_pipeline_flyout.tsx new file mode 100644 index 0000000000000..7d4ea408111fe --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/add_inference_pipeline_flyout.tsx @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useMemo, useState } from 'react'; + +import { + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutHeader, + EuiFlyoutFooter, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { extractErrorProperties } from '@kbn/ml-error-utils'; + +import { ModelItem } from '../../model_management/models_list'; +import type { AddInferencePipelineSteps } from './types'; +import { ADD_INFERENCE_PIPELINE_STEPS } from './constants'; +import { AddInferencePipelineFooter } from './components/add_inference_pipeline_footer'; +import { AddInferencePipelineHorizontalSteps } from './components/add_inference_pipeline_horizontal_steps'; +import { getInitialState, getModelType } from './state'; +import { PipelineDetails } from './components/pipeline_details'; +import { ProcessorConfiguration } from './components/processor_configuration'; +import { OnFailureConfiguration } from './components/on_failure_configuration'; +import { TestPipeline } from './components/test_pipeline'; +import { ReviewAndCreatePipeline } from './components/review_and_create_pipeline'; +import { useMlApiContext } from '../../contexts/kibana'; +import { getPipelineConfig } from './get_pipeline_config'; +import { validateInferencePipelineConfigurationStep } from './validation'; +import type { MlInferenceState, InferenceModelTypes } from './types'; +import { useFetchPipelines } from './hooks/use_fetch_pipelines'; + +export interface AddInferencePipelineFlyoutProps { + onClose: () => void; + model: ModelItem; +} + +export const AddInferencePipelineFlyout: FC = ({ + onClose, + model, +}) => { + // eslint-disable-next-line react-hooks/exhaustive-deps + const initialState = useMemo(() => getInitialState(model), [model.model_id]); + const [formState, setFormState] = useState(initialState); + const [step, setStep] = useState(ADD_INFERENCE_PIPELINE_STEPS.DETAILS); + const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false); + + const { + trainedModels: { createInferencePipeline }, + } = useMlApiContext(); + + const modelType = getModelType(model); + + const createPipeline = async () => { + setFormState({ ...formState, creatingPipeline: true }); + try { + await createInferencePipeline(formState.pipelineName, getPipelineConfig(formState)); + setFormState({ + ...formState, + pipelineCreated: true, + creatingPipeline: false, + pipelineError: undefined, + }); + } catch (e) { + // eslint-disable-next-line no-console + console.error(e); + const errorProperties = extractErrorProperties(e); + setFormState({ + ...formState, + creatingPipeline: false, + pipelineError: errorProperties.message ?? e.message, + }); + } + }; + + const pipelineNames = useFetchPipelines(); + + const handleConfigUpdate = (configUpdate: Partial) => { + setFormState({ ...formState, ...configUpdate }); + }; + + const { pipelineName: pipelineNameError, targetField: targetFieldError } = useMemo(() => { + const errors = validateInferencePipelineConfigurationStep( + formState.pipelineName, + pipelineNames + ); + return errors; + }, [pipelineNames, formState.pipelineName]); + + const sourceIndex = useMemo( + () => + Array.isArray(model.metadata?.analytics_config.source.index) + ? model.metadata?.analytics_config.source.index.join() + : model.metadata?.analytics_config.source.index, + // eslint-disable-next-line react-hooks/exhaustive-deps + [model?.model_id] + ); + + return ( + + + +

+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.title', + { + defaultMessage: 'Deploy analytics model', + } + )} +

+ + + + + + {step === ADD_INFERENCE_PIPELINE_STEPS.DETAILS && ( + + )} + {step === ADD_INFERENCE_PIPELINE_STEPS.CONFIGURE_PROCESSOR && model && ( + + )} + {step === ADD_INFERENCE_PIPELINE_STEPS.ON_FAILURE && ( + + )} + {step === ADD_INFERENCE_PIPELINE_STEPS.TEST && ( + + )} + {step === ADD_INFERENCE_PIPELINE_STEPS.CREATE && ( + + )} + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_footer.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_footer.tsx new file mode 100644 index 0000000000000..f0a8beb2482f6 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_footer.tsx @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; + +import { EuiButton, EuiButtonEmpty, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { AddInferencePipelineSteps } from '../types'; +import { + BACK_BUTTON_LABEL, + CANCEL_BUTTON_LABEL, + CLOSE_BUTTON_LABEL, + CONTINUE_BUTTON_LABEL, +} from '../constants'; +import { getSteps } from '../get_steps'; + +interface Props { + isDetailsStepValid: boolean; + isConfigureProcessorStepValid: boolean; + pipelineCreated: boolean; + creatingPipeline: boolean; + step: AddInferencePipelineSteps; + onClose: () => void; + onCreate: () => void; + setStep: React.Dispatch>; +} + +export const AddInferencePipelineFooter: FC = ({ + isDetailsStepValid, + isConfigureProcessorStepValid, + creatingPipeline, + pipelineCreated, + onClose, + onCreate, + step, + setStep, +}) => { + const { nextStep, previousStep, isContinueButtonEnabled } = useMemo( + () => getSteps(step, isDetailsStepValid, isConfigureProcessorStepValid), + [isDetailsStepValid, isConfigureProcessorStepValid, step] + ); + + return ( + + + + {pipelineCreated ? CLOSE_BUTTON_LABEL : CANCEL_BUTTON_LABEL} + + + + + {previousStep !== undefined && pipelineCreated === false ? ( + setStep(previousStep as AddInferencePipelineSteps)} + > + {BACK_BUTTON_LABEL} + + ) : null} + + + {nextStep !== undefined ? ( + setStep(nextStep as AddInferencePipelineSteps)} + disabled={!isContinueButtonEnabled} + fill + > + {CONTINUE_BUTTON_LABEL} + + ) : ( + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.addInferencePipelineModal.footer.create', + { + defaultMessage: 'Create pipeline', + } + )} + + )} + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_horizontal_steps.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_horizontal_steps.tsx new file mode 100644 index 0000000000000..9954ed8955259 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/add_inference_pipeline_horizontal_steps.tsx @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, memo } from 'react'; +import { i18n } from '@kbn/i18n'; + +import { EuiStepsHorizontal, EuiStepsHorizontalProps } from '@elastic/eui'; +import type { AddInferencePipelineSteps } from '../types'; +import { ADD_INFERENCE_PIPELINE_STEPS } from '../constants'; + +const steps = Object.values(ADD_INFERENCE_PIPELINE_STEPS); + +interface Props { + step: AddInferencePipelineSteps; + setStep: React.Dispatch>; + isDetailsStepValid: boolean; + isConfigureProcessorStepValid: boolean; +} + +export const AddInferencePipelineHorizontalSteps: FC = memo( + ({ step, setStep, isDetailsStepValid, isConfigureProcessorStepValid }) => { + const currentStepIndex = steps.findIndex((s) => s === step); + const navSteps: EuiStepsHorizontalProps['steps'] = [ + { + // Details + onClick: () => setStep(ADD_INFERENCE_PIPELINE_STEPS.DETAILS), + status: isDetailsStepValid ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.ml.inferencePipeline.content.indices.transforms.addInferencePipelineModal.steps.details.title', + { + defaultMessage: 'Details', + } + ), + }, + { + // Processor configuration + onClick: () => { + if (!isDetailsStepValid) return; + setStep(ADD_INFERENCE_PIPELINE_STEPS.CONFIGURE_PROCESSOR); + }, + status: + isDetailsStepValid && isConfigureProcessorStepValid && currentStepIndex > 1 + ? 'complete' + : 'incomplete', + title: i18n.translate( + 'xpack.ml.inferencePipeline.content.indices.transforms.addInferencePipelineModal.steps.configureProcessor.title', + { + defaultMessage: 'Configure processor', + } + ), + }, + { + // handle failures + onClick: () => { + if (!isDetailsStepValid) return; + setStep(ADD_INFERENCE_PIPELINE_STEPS.ON_FAILURE); + }, + status: currentStepIndex > 2 ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.ml.inferencePipeline.content.indices.transforms.addInferencePipelineModal.steps.handleFailures.title', + { + defaultMessage: 'Handle failures', + } + ), + }, + { + // Test + onClick: () => { + if (!isConfigureProcessorStepValid || !isDetailsStepValid) return; + setStep(ADD_INFERENCE_PIPELINE_STEPS.TEST); + }, + status: currentStepIndex > 3 ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.ml.trainedModels.content.indices.transforms.addInferencePipelineModal.steps.test.title', + { + defaultMessage: 'Test (Optional)', + } + ), + }, + { + // Review and Create + onClick: () => { + if (!isConfigureProcessorStepValid) return; + setStep(ADD_INFERENCE_PIPELINE_STEPS.CREATE); + }, + status: isDetailsStepValid && isConfigureProcessorStepValid ? 'incomplete' : 'disabled', + title: i18n.translate( + 'xpack.ml.inferencePipeline.content.indices.transforms.addInferencePipelineModal.steps.create.title', + { + defaultMessage: 'Create', + } + ), + }, + ]; + switch (step) { + case ADD_INFERENCE_PIPELINE_STEPS.DETAILS: + navSteps[0].status = 'current'; + break; + case ADD_INFERENCE_PIPELINE_STEPS.CONFIGURE_PROCESSOR: + navSteps[1].status = 'current'; + break; + case ADD_INFERENCE_PIPELINE_STEPS.ON_FAILURE: + navSteps[2].status = 'current'; + break; + case ADD_INFERENCE_PIPELINE_STEPS.TEST: + navSteps[3].status = 'current'; + break; + case ADD_INFERENCE_PIPELINE_STEPS.CREATE: + navSteps[4].status = 'current'; + break; + } + return ; + } +); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/additional_advanced_settings.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/additional_advanced_settings.tsx new file mode 100644 index 0000000000000..4b3cfbfcf795b --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/additional_advanced_settings.tsx @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useState, memo, useMemo } from 'react'; + +import { + EuiAccordion, + EuiFlexGroup, + EuiFieldText, + EuiFlexItem, + EuiFormRow, + EuiLink, + EuiPanel, + EuiTextArea, + htmlIdGenerator, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import type { AdditionalSettings, MlInferenceState } from '../types'; +import { SaveChangesButton } from './save_changes_button'; +import { useMlKibana } from '../../../contexts/kibana'; + +interface Props { + condition?: string; + tag?: string; + handleAdvancedConfigUpdate: (configUpdate: Partial) => void; +} + +export const AdditionalAdvancedSettings: FC = memo( + ({ handleAdvancedConfigUpdate, condition, tag }) => { + const [additionalSettings, setAdditionalSettings] = useState< + Partial | undefined + >(condition || tag ? { condition, tag } : undefined); + + const { + services: { + docLinks: { links }, + }, + } = useMlKibana(); + + const handleAdditionalSettingsChange = (settingsChange: Partial) => { + setAdditionalSettings({ ...additionalSettings, ...settingsChange }); + }; + + const accordionId = useMemo(() => htmlIdGenerator()(), []); + const additionalSettingsUpdated = useMemo( + () => additionalSettings?.tag !== tag || additionalSettings?.condition !== condition, + [additionalSettings, tag, condition] + ); + + const updateAdditionalSettings = () => { + handleAdvancedConfigUpdate({ ...additionalSettings }); + }; + + return ( + + + + + + {additionalSettingsUpdated ? ( + + ) : null} + + + } + > + + + {/* CONDITION */} + + + } + helpText={ + + Painless + + ), + }} + /> + } + > + ) => + handleAdditionalSettingsChange({ condition: e.target.value }) + } + /> + + + {/* TAG */} + + + + + } + helpText={ + + } + > + ) => + handleAdditionalSettingsChange({ tag: e.target.value }) + } + aria-label={i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.tagAriaLabel', + { defaultMessage: 'Optional tag identifier for the processor' } + )} + /> + + + + + + + + ); + } +); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/on_failure_configuration.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/on_failure_configuration.tsx new file mode 100644 index 0000000000000..bc8bc4eedb2d3 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/on_failure_configuration.tsx @@ -0,0 +1,270 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useState, memo } from 'react'; + +import { + EuiButtonEmpty, + EuiCode, + EuiCodeBlock, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiLink, + EuiSpacer, + EuiSwitch, + EuiSwitchEvent, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { CodeEditor } from '@kbn/kibana-react-plugin/public'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { SaveChangesButton } from './save_changes_button'; +import type { MlInferenceState } from '../types'; +import { getDefaultOnFailureConfiguration } from '../state'; +import { CANCEL_EDIT_MESSAGE, EDIT_MESSAGE } from '../constants'; +import { useMlKibana } from '../../../contexts/kibana'; +import { isValidJson } from '../../../../../common/util/validation_utils'; + +interface Props { + handleAdvancedConfigUpdate: (configUpdate: Partial) => void; + ignoreFailure: boolean; + onFailure: MlInferenceState['onFailure']; + takeActionOnFailure: MlInferenceState['takeActionOnFailure']; +} + +export const OnFailureConfiguration: FC = memo( + ({ handleAdvancedConfigUpdate, ignoreFailure, onFailure, takeActionOnFailure }) => { + const { + services: { + docLinks: { links }, + }, + } = useMlKibana(); + + const [editOnFailure, setEditOnFailure] = useState(false); + const [isOnFailureValid, setIsOnFailureValid] = useState(false); + const [onFailureString, setOnFailureString] = useState( + JSON.stringify(onFailure, null, 2) + ); + + const updateIgnoreFailure = (e: EuiSwitchEvent) => { + const checked = e.target.checked; + handleAdvancedConfigUpdate({ + ignoreFailure: checked, + ...(checked === true ? { takeActionOnFailure: false, onFailure: undefined } : {}), + }); + }; + + const updateOnFailure = () => { + handleAdvancedConfigUpdate({ onFailure: JSON.parse(onFailureString) }); + setEditOnFailure(false); + }; + + const handleOnFailureChange = (json: string) => { + setOnFailureString(json); + const valid = isValidJson(json); + setIsOnFailureValid(valid); + }; + + const handleTakeActionOnFailureChange = (checked: boolean) => { + handleAdvancedConfigUpdate({ + takeActionOnFailure: checked, + onFailure: checked === false ? undefined : getDefaultOnFailureConfiguration(), + }); + if (checked === false) { + setEditOnFailure(false); + setIsOnFailureValid(true); + } + }; + + const resetOnFailure = () => { + setOnFailureString(JSON.stringify(getDefaultOnFailureConfiguration(), null, 2)); + setIsOnFailureValid(true); + }; + + return ( + + + + + +

+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.onFailureTitle', + { defaultMessage: 'Ingesting problematic documents' } + )} +

+ + + +

+ +

+

+ {'ignore_failure'}, + inferenceDocsLink: ( + + Learn more. + + ), + }} + /> +

+

+ {'on_failure'}, + onFailureDocsLink: ( + + Learn more. + + ), + }} + /> +

+ + + + + + + + + + } + checked={ignoreFailure} + onChange={updateIgnoreFailure} + /> + + + + {ignoreFailure === false ? ( + + + + + } + checked={takeActionOnFailure} + onChange={(e: EuiSwitchEvent) => + handleTakeActionOnFailureChange(e.target.checked) + } + /> + + ) : null} + + + + + {takeActionOnFailure === true && ignoreFailure === false ? ( + + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.onFailureHeadingLabel', + { defaultMessage: 'Actions to take on failure' } + )} + + + } + labelAppend={ + + + { + setEditOnFailure(!editOnFailure); + }} + > + {editOnFailure ? CANCEL_EDIT_MESSAGE : EDIT_MESSAGE} + + + + {editOnFailure ? ( + + ) : null} + + + {editOnFailure ? ( + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.resetOnFailureButton', + { defaultMessage: 'Reset' } + )} + + ) : null} + + + } + helpText={ + + } + > + <> + {!editOnFailure ? ( + + {JSON.stringify(onFailure, null, 2)} + + ) : null} + {editOnFailure ? ( + + ) : null} + + + ) : null} + + + + + + + ); + } +); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/pipeline_details.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/pipeline_details.tsx new file mode 100644 index 0000000000000..4988c772c2863 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/pipeline_details.tsx @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, memo } from 'react'; + +import { + EuiCode, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiForm, + EuiFormRow, + EuiLink, + EuiSpacer, + EuiTitle, + EuiText, + EuiTextArea, + EuiPanel, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { useMlKibana } from '../../../contexts/kibana'; +import type { MlInferenceState } from '../types'; + +interface Props { + handlePipelineConfigUpdate: (configUpdate: Partial) => void; + modelId: string; + pipelineNameError: string | undefined; + pipelineName: string; + pipelineDescription: string; + targetField: string; + targetFieldError: string | undefined; +} + +export const PipelineDetails: FC = memo( + ({ + handlePipelineConfigUpdate, + modelId, + pipelineName, + pipelineNameError, + pipelineDescription, + targetField, + targetFieldError, + }) => { + const { + services: { + docLinks: { links }, + }, + } = useMlKibana(); + + const handleConfigChange = (value: string, type: string) => { + handlePipelineConfigUpdate({ [type]: value }); + }; + + return ( + + + +

+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.configure.title', + { defaultMessage: 'Create a pipeline' } + )} +

+ + + +

+ {modelId}, + pipeline: ( + + pipeline + + ), + }} + /> +

+

+ + _reindex API + + ), + pipelineSimulateLink: ( + + pipeline/_simulate + + ), + }} + /> +

+ + + + + {/* NAME */} + + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText', + { + defaultMessage: + 'Pipeline names are unique within a deployment and can only contain letters, numbers, underscores, and hyphens.', + } + )} + + ) + } + error={pipelineNameError} + isInvalid={pipelineNameError !== undefined} + > + ) => + handleConfigChange(e.target.value, 'pipelineName') + } + /> + + {/* DESCRIPTION */} + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.configure.description.helpText', + { + defaultMessage: 'A description of what this pipeline does.', + } + )} + + } + > + ) => + handleConfigChange(e.target.value, 'pipelineDescription') + } + /> + + {/* TARGET FIELD */} + {'ml.inference.'} }} + /> + ) + } + error={targetFieldError} + isInvalid={targetFieldError !== undefined} + > + ) => + handleConfigChange(e.target.value, 'targetField') + } + /> + + + + + + ); + } +); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/processor_configuration.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/processor_configuration.tsx new file mode 100644 index 0000000000000..7f2dfe9ede728 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/processor_configuration.tsx @@ -0,0 +1,391 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useState, memo } from 'react'; + +import { + EuiButtonEmpty, + EuiCode, + EuiCodeBlock, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiLink, + EuiSpacer, + EuiText, + EuiTitle, + EuiPopover, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { CodeEditor } from '@kbn/kibana-react-plugin/public'; +import { ModelItem } from '../../../model_management/models_list'; +import { + EDIT_MESSAGE, + CANCEL_EDIT_MESSAGE, + CREATE_FIELD_MAPPING_MESSAGE, + CLEAR_BUTTON_LABEL, +} from '../constants'; +import { validateInferenceConfig } from '../validation'; +import { isValidJson } from '../../../../../common/util/validation_utils'; +import { SaveChangesButton } from './save_changes_button'; +import { useMlKibana } from '../../../contexts/kibana'; +import type { MlInferenceState, InferenceModelTypes } from '../types'; +import { AdditionalAdvancedSettings } from './additional_advanced_settings'; +import { validateFieldMap } from '../validation'; + +function getDefaultFieldMapString() { + return JSON.stringify( + { + field_map: { + incoming_field: 'field_the_model_expects', + }, + }, + null, + 2 + ); +} + +interface Props { + condition?: string; + fieldMap: MlInferenceState['fieldMap']; + handleAdvancedConfigUpdate: (configUpdate: Partial) => void; + inferenceConfig: ModelItem['inference_config']; + modelInferenceConfig: ModelItem['inference_config']; + modelInputFields: ModelItem['input']; + modelType?: InferenceModelTypes; + setHasUnsavedChanges: React.Dispatch>; + tag?: string; +} + +export const ProcessorConfiguration: FC = memo( + ({ + condition, + fieldMap, + handleAdvancedConfigUpdate, + inferenceConfig, + modelInputFields, + modelInferenceConfig, + modelType, + setHasUnsavedChanges, + tag, + }) => { + const { + services: { + docLinks: { links }, + }, + } = useMlKibana(); + + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const [editInferenceConfig, setEditInferenceConfig] = useState(false); + const [editFieldMapping, setEditFieldMapping] = useState(false); + const [inferenceConfigString, setInferenceConfigString] = useState( + JSON.stringify(inferenceConfig, null, 2) + ); + const [inferenceConfigError, setInferenceConfigError] = useState(); + const [fieldMapError, setFieldMapError] = useState(); + const [fieldMappingString, setFieldMappingString] = useState( + fieldMap ? JSON.stringify(fieldMap, null, 2) : getDefaultFieldMapString() + ); + const [isInferenceConfigValid, setIsInferenceConfigValid] = useState(true); + const [isFieldMapValid, setIsFieldMapValid] = useState(true); + + const handleInferenceConfigChange = (json: string) => { + setInferenceConfigString(json); + const valid = isValidJson(json); + setIsInferenceConfigValid(valid); + }; + + const updateInferenceConfig = () => { + const invalidInferenceConfigMessage = validateInferenceConfig( + JSON.parse(inferenceConfigString), + modelType + ); + + if (invalidInferenceConfigMessage === undefined) { + handleAdvancedConfigUpdate({ inferenceConfig: JSON.parse(inferenceConfigString) }); + setHasUnsavedChanges(false); + setEditInferenceConfig(false); + setInferenceConfigError(undefined); + } else { + setHasUnsavedChanges(true); + setIsInferenceConfigValid(false); + setInferenceConfigError(invalidInferenceConfigMessage); + } + }; + + const resetInferenceConfig = () => { + setInferenceConfigString(JSON.stringify(modelInferenceConfig, null, 2)); + setIsInferenceConfigValid(true); + setInferenceConfigError(undefined); + }; + + const clearFieldMap = () => { + setFieldMappingString('{}'); + setIsFieldMapValid(true); + setFieldMapError(undefined); + }; + + const handleFieldMapChange = (json: string) => { + setFieldMappingString(json); + const valid = isValidJson(json); + setIsFieldMapValid(valid); + }; + + const updateFieldMap = () => { + const invalidFieldMapMessage = validateFieldMap( + modelInputFields.field_names ?? [], + JSON.parse(fieldMappingString) + ); + + if (invalidFieldMapMessage === undefined) { + handleAdvancedConfigUpdate({ fieldMap: JSON.parse(fieldMappingString) }); + setHasUnsavedChanges(false); + setEditFieldMapping(false); + setFieldMapError(undefined); + } else { + setHasUnsavedChanges(true); + setIsFieldMapValid(false); + setFieldMapError(invalidFieldMapMessage); + } + }; + + return ( + + {/* INFERENCE CONFIG */} + + + + +

+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.inferenceConfigurationTitle', + { defaultMessage: 'Inference configuration' } + )} +

+ + + +

+ + Learn more. + + ), + }} + /> +

+ + + + + + { + if (!editInferenceConfig === false) { + setInferenceConfigError(undefined); + setIsInferenceConfigValid(true); + } + setEditInferenceConfig(!editInferenceConfig); + }} + > + {editInferenceConfig ? CANCEL_EDIT_MESSAGE : EDIT_MESSAGE} + + + + {editInferenceConfig ? ( + + ) : null} + + + {editInferenceConfig ? ( + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.resetInferenceConfigButton', + { defaultMessage: 'Reset' } + )} + + ) : null} + + + } + error={inferenceConfigError ?? inferenceConfigError} + isInvalid={inferenceConfigError !== undefined || inferenceConfigError !== undefined} + > + {editInferenceConfig ? ( + + ) : ( + + {JSON.stringify(inferenceConfig, null, 2)} + + )} + + + + + {/* FIELD MAP */} + + + + +

+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.fieldMapTitle', + { defaultMessage: 'Fields' } + )} +

+ + + +

+ setIsPopoverOpen(!isPopoverOpen)}> + You can review them here. + + } + isOpen={isPopoverOpen} + closePopover={() => setIsPopoverOpen(false)} + anchorPosition="downLeft" + > + + {JSON.stringify(modelInputFields, null, 2)} + + + ), + }} + /> +

+

+ {'field_map'}, + inferenceDocsLink: ( + + Learn more. + + ), + }} + /> +

+ + + + + + { + const editingState = !editFieldMapping; + if (editingState === false) { + setFieldMapError(undefined); + setIsFieldMapValid(true); + setHasUnsavedChanges(false); + } + setEditFieldMapping(editingState); + }} + > + {editFieldMapping + ? CANCEL_EDIT_MESSAGE + : fieldMap !== undefined + ? EDIT_MESSAGE + : CREATE_FIELD_MAPPING_MESSAGE} + + + {editFieldMapping ? ( + + + + ) : null} + {editFieldMapping ? ( + + + {CLEAR_BUTTON_LABEL} + + + ) : null} + + } + error={fieldMapError} + isInvalid={fieldMapError !== undefined} + > + <> + {!editFieldMapping ? ( + + {JSON.stringify(fieldMap ?? {}, null, 2)} + + ) : null} + {editFieldMapping ? ( + <> + + + + ) : null} + + + + + + {/* ADDITIONAL ADVANCED SETTINGS */} + + + + + ); + } +); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/review_and_create_pipeline.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/review_and_create_pipeline.tsx new file mode 100644 index 0000000000000..352f11a0ba867 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/review_and_create_pipeline.tsx @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useMemo } from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { + EuiAccordion, + EuiCallOut, + EuiCodeBlock, + EuiFlexGroup, + EuiFlexItem, + EuiLink, + EuiSpacer, + EuiTitle, + EuiText, + htmlIdGenerator, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; +import { useMlKibana } from '../../../contexts/kibana'; + +const MANAGEMENT_APP_ID = 'management'; + +interface Props { + inferencePipeline: IngestPipeline; + modelType?: string; + pipelineName: string; + pipelineCreated: boolean; + pipelineError?: string; +} + +export const ReviewAndCreatePipeline: FC = ({ + inferencePipeline, + modelType, + pipelineName, + pipelineCreated, + pipelineError, +}) => { + const { + services: { + application, + docLinks: { links }, + }, + } = useMlKibana(); + + const inferenceProcessorLink = + modelType === 'regression' + ? links.ingest.inferenceRegression + : links.ingest.inferenceClassification; + + const accordionId = useMemo(() => htmlIdGenerator()(), []); + + const configCodeBlock = useMemo( + () => ( + + {JSON.stringify(inferencePipeline ?? {}, null, 2)} + + ), + [inferencePipeline] + ); + + return ( + <> + + + {pipelineCreated === false ? ( + +

+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.review.title', + { + defaultMessage: "Review the pipeline configuration for '{pipelineName}'", + values: { pipelineName }, + } + )} +

+ + ) : null} + <> + + {pipelineCreated === true && pipelineError === undefined ? ( + +

+ + {'reindexing'} + + ), + }} + /> + {application.capabilities.management?.ingest?.ingest_pipelines ? ( + { + await application.navigateToApp(MANAGEMENT_APP_ID, { + path: `/ingest/ingest_pipelines/?pipeline=${pipelineName}`, + openInNewTab: true, + }); + }} + target="_blank" + external + > + {'Ingest Pipelines'} + + ), + }} + /> + ) : null} +

+ + ) : null} + {pipelineError !== undefined ? ( + +

{pipelineError}

+

+ + {'ingest pipeline'} + + ), + inferencePipelineConfigLink: ( + + {'inference processor'} + + ), + }} + /> +

+ + ) : null} + + + + +

+ {!pipelineCreated ? ( + + ) : null} +

+ + + + {pipelineCreated ? ( + <> + + + } + > + {configCodeBlock} + + + ) : ( + [configCodeBlock] + )} + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/save_changes_button.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/save_changes_button.tsx new file mode 100644 index 0000000000000..73e763d153799 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/save_changes_button.tsx @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC } from 'react'; +import { EuiButtonEmpty } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +interface SaveChangesButtonProps { + onClick: () => void; + disabled: boolean; +} + +export const SaveChangesButton: FC = ({ onClick, disabled }) => ( + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.saveChangesButton', + { defaultMessage: 'Save changes' } + )} + +); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/components/test_pipeline.tsx b/x-pack/plugins/ml/public/application/components/ml_inference/components/test_pipeline.tsx new file mode 100644 index 0000000000000..19120de441f4c --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/components/test_pipeline.tsx @@ -0,0 +1,279 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, memo, useEffect, useCallback, useState } from 'react'; +import { css } from '@emotion/react'; +import { euiThemeVars } from '@kbn/ui-theme'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { + EuiButton, + EuiButtonEmpty, + EuiCode, + EuiCodeBlock, + EuiFlexGroup, + EuiFlexItem, + EuiResizableContainer, + EuiSpacer, + EuiTitle, + EuiText, + useIsWithinMaxBreakpoint, + EuiPanel, +} from '@elastic/eui'; + +import { IngestSimulateDocument } from '@elastic/elasticsearch/lib/api/types'; +import { extractErrorProperties } from '@kbn/ml-error-utils'; + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { CodeEditor } from '@kbn/kibana-react-plugin/public'; +import { useMlApiContext, useMlKibana } from '../../../contexts/kibana'; +import { getPipelineConfig } from '../get_pipeline_config'; +import { isValidJson } from '../../../../../common/util/validation_utils'; +import type { MlInferenceState } from '../types'; + +interface Props { + sourceIndex?: string; + state: MlInferenceState; +} + +export const TestPipeline: FC = memo(({ state, sourceIndex }) => { + const [simulatePipelineResult, setSimulatePipelineResult] = useState< + undefined | estypes.IngestSimulateResponse + >(); + const [simulatePipelineError, setSimulatePipelineError] = useState(); + const [sampleDocsString, setSampleDocsString] = useState(''); + const [isValid, setIsValid] = useState(true); + const { + esSearch, + trainedModels: { trainedModelPipelineSimulate }, + } = useMlApiContext(); + const { + notifications: { toasts }, + } = useMlKibana(); + + const isSmallerViewport = useIsWithinMaxBreakpoint('s'); + + const simulatePipeline = async () => { + try { + const pipelineConfig = getPipelineConfig(state); + const result = await trainedModelPipelineSimulate( + pipelineConfig, + JSON.parse(sampleDocsString) as IngestSimulateDocument[] + ); + setSimulatePipelineResult(result); + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + const errorProperties = extractErrorProperties(error); + setSimulatePipelineError(error); + toasts.danger({ + title: i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.errorSimulatingPipeline', + { + defaultMessage: 'Unable to simulate pipeline.', + } + ), + body: errorProperties.message, + toastLifeTimeMs: 5000, + }); + } + }; + + const clearResults = () => { + setSimulatePipelineResult(undefined); + setSimulatePipelineError(undefined); + }; + + const onChange = (json: string) => { + setSampleDocsString(json); + const valid = isValidJson(json); + setIsValid(valid); + }; + + const getSampleDocs = useCallback(async () => { + let records: IngestSimulateDocument[] = []; + let resp; + + try { + resp = await esSearch({ + index: sourceIndex, + body: { + size: 1, + }, + }); + + if (resp && resp.hits.total.value > 0) { + records = resp.hits.hits; + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } + setSampleDocsString(JSON.stringify(records, null, 2)); + setIsValid(true); + }, [sourceIndex, esSearch]); + + useEffect( + function fetchSampleDocsFromSource() { + if (sourceIndex) { + getSampleDocs(); + } + }, + [sourceIndex, getSampleDocs] + ); + + return ( + <> + + + +

+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.title', + { defaultMessage: 'Test the pipeline results' } + )} +

+ + + + +

+ + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.optionalCallout', + { defaultMessage: 'This is an optional step.' } + )} + +   + {' '} + {state.targetField && ( + {state.targetField} }} + /> + )} +

+ + + + + + + + + +
+ + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.runButton', + { defaultMessage: 'Simulate pipeline' } + )} + +
+ + + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.clearResultsButton', + { defaultMessage: 'Clear results' } + )} + + + + + {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.resetSampleDocsButton', + { defaultMessage: 'Reset sample docs' } + )} + + + + + + + + + +
+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.subtitle.documents', + { defaultMessage: 'Raw document' } + )} +
+ + + + +
+ {i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.test.subtitle.result', + { defaultMessage: 'Result' } + )} +
+ + + + + + + {(EuiResizablePanel, EuiResizableButton) => ( + <> + + + + + + + + + {simulatePipelineError + ? JSON.stringify(simulatePipelineError, null, 2) + : simulatePipelineResult + ? JSON.stringify(simulatePipelineResult, null, 2) + : '{}'} + + + + )} + + + + + + + ); +}); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/constants.ts b/x-pack/plugins/ml/public/application/components/ml_inference/constants.ts new file mode 100644 index 0000000000000..8bf26c55b1b6d --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/constants.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const ADD_INFERENCE_PIPELINE_STEPS = { + DETAILS: 'Details', + CONFIGURE_PROCESSOR: 'Configure processor', + ON_FAILURE: 'Failure handling', + TEST: 'Test', + CREATE: 'create', +} as const; + +export const CANCEL_BUTTON_LABEL = i18n.translate( + 'xpack.ml.trainedModels.actions.cancelButtonLabel', + { defaultMessage: 'Cancel' } +); + +export const CLEAR_BUTTON_LABEL = i18n.translate( + 'xpack.ml.trainedModels.actions.clearButtonLabel', + { defaultMessage: 'Clear' } +); + +export const CLOSE_BUTTON_LABEL = i18n.translate( + 'xpack.ml.trainedModels.actions.closeButtonLabel', + { defaultMessage: 'Close' } +); + +export const BACK_BUTTON_LABEL = i18n.translate('xpack.ml.trainedModels.actions.backButtonLabel', { + defaultMessage: 'Back', +}); + +export const CONTINUE_BUTTON_LABEL = i18n.translate( + 'xpack.ml.trainedModels.actions.continueButtonLabel', + { defaultMessage: 'Continue' } +); + +export const EDIT_MESSAGE = i18n.translate( + 'xpack.ml.trainedModels.actions.create.advancedDetails.editButtonText', + { + defaultMessage: 'Edit', + } +); + +export const CREATE_FIELD_MAPPING_MESSAGE = i18n.translate( + 'xpack.ml.trainedModels.actions.create.advancedDetails.createFieldMapText', + { + defaultMessage: 'Create field map', + } +); + +export const CANCEL_EDIT_MESSAGE = i18n.translate( + 'xpack.ml.trainedModels.actions.create.advancedDetails.cancelEditButtonText', + { + defaultMessage: 'Cancel', + } +); diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/get_pipeline_config.ts b/x-pack/plugins/ml/public/application/components/ml_inference/get_pipeline_config.ts new file mode 100644 index 0000000000000..d7752a069150e --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/get_pipeline_config.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MlInferenceState } from './types'; + +export function getPipelineConfig(state: MlInferenceState) { + const { + condition, + fieldMap, + ignoreFailure, + inferenceConfig, + modelId, + onFailure, + pipelineDescription, + tag, + targetField, + } = state; + return { + description: pipelineDescription, + processors: [ + { + inference: { + model_id: modelId, + ignore_failure: ignoreFailure, + ...(targetField && targetField !== '' ? { target_field: targetField } : {}), + ...(fieldMap && Object.keys(fieldMap).length > 0 ? { field_map: fieldMap } : {}), + ...(inferenceConfig && Object.keys(inferenceConfig).length > 0 + ? { inference_config: inferenceConfig } + : {}), + ...(condition && condition !== '' ? { if: condition } : {}), + ...(tag && tag !== '' ? { tag } : {}), + ...(onFailure && Object.keys(onFailure).length > 0 ? { on_failure: onFailure } : {}), + }, + }, + ], + }; +} diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/get_steps.ts b/x-pack/plugins/ml/public/application/components/ml_inference/get_steps.ts new file mode 100644 index 0000000000000..a7d3ea17de099 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/get_steps.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AddInferencePipelineSteps } from './types'; +import { ADD_INFERENCE_PIPELINE_STEPS } from './constants'; + +export function getSteps( + step: AddInferencePipelineSteps, + isConfigureStepValid: boolean, + isPipelineDataValid: boolean +) { + let nextStep: AddInferencePipelineSteps | undefined; + let previousStep: AddInferencePipelineSteps | undefined; + let isContinueButtonEnabled = false; + + switch (step) { + case ADD_INFERENCE_PIPELINE_STEPS.DETAILS: + nextStep = ADD_INFERENCE_PIPELINE_STEPS.CONFIGURE_PROCESSOR; + isContinueButtonEnabled = isConfigureStepValid; + break; + case ADD_INFERENCE_PIPELINE_STEPS.CONFIGURE_PROCESSOR: + nextStep = ADD_INFERENCE_PIPELINE_STEPS.ON_FAILURE; + previousStep = ADD_INFERENCE_PIPELINE_STEPS.DETAILS; + isContinueButtonEnabled = isPipelineDataValid; + break; + case ADD_INFERENCE_PIPELINE_STEPS.ON_FAILURE: + nextStep = ADD_INFERENCE_PIPELINE_STEPS.TEST; + previousStep = ADD_INFERENCE_PIPELINE_STEPS.CONFIGURE_PROCESSOR; + isContinueButtonEnabled = isPipelineDataValid; + break; + case ADD_INFERENCE_PIPELINE_STEPS.TEST: + nextStep = ADD_INFERENCE_PIPELINE_STEPS.CREATE; + previousStep = ADD_INFERENCE_PIPELINE_STEPS.ON_FAILURE; + isContinueButtonEnabled = true; + break; + case ADD_INFERENCE_PIPELINE_STEPS.CREATE: + previousStep = ADD_INFERENCE_PIPELINE_STEPS.TEST; + isContinueButtonEnabled = true; + break; + } + + return { nextStep, previousStep, isContinueButtonEnabled }; +} diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/hooks/use_fetch_pipelines.ts b/x-pack/plugins/ml/public/application/components/ml_inference/hooks/use_fetch_pipelines.ts new file mode 100644 index 0000000000000..837aef2f92093 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/hooks/use_fetch_pipelines.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { useMlApiContext, useMlKibana } from '../../../contexts/kibana'; + +export const useFetchPipelines = () => { + const [pipelineNames, setPipelineNames] = useState([]); + const { + notifications: { toasts }, + } = useMlKibana(); + + const { + trainedModels: { getAllIngestPipelines }, + } = useMlApiContext(); + + useEffect(() => { + async function fetchPipelines() { + let names: string[] = []; + try { + const results = await getAllIngestPipelines(); + names = Object.keys(results); + setPipelineNames(names); + } catch (e) { + toasts.danger({ + title: i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.fetchIngestPipelinesError', + { + defaultMessage: 'Unable to fetch ingest pipelines.', + } + ), + body: e.message, + toastLifeTimeMs: 5000, + }); + } + } + + fetchPipelines(); + }, [getAllIngestPipelines, toasts]); + + return pipelineNames; +}; diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/index.ts b/x-pack/plugins/ml/public/application/components/ml_inference/index.ts new file mode 100644 index 0000000000000..0c079b0273e98 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { AddInferencePipelineFlyout } from './add_inference_pipeline_flyout'; diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/state.ts b/x-pack/plugins/ml/public/application/components/ml_inference/state.ts new file mode 100644 index 0000000000000..6c74e279fa147 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/state.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getAnalysisType } from '@kbn/ml-data-frame-analytics-utils'; +import type { MlInferenceState } from './types'; +import { ModelItem } from '../../model_management/models_list'; + +export const getModelType = (model: ModelItem): string | undefined => { + const analysisConfig = model.metadata?.analytics_config?.analysis; + return analysisConfig !== undefined ? getAnalysisType(analysisConfig) : undefined; +}; + +export const getDefaultOnFailureConfiguration = (): MlInferenceState['onFailure'] => [ + { + set: { + description: "Index document to 'failed-'", + field: '_index', + value: 'failed-{{{ _index }}}', + }, + }, + { + set: { + field: 'event.timestamp', + value: '{{{ _ingest.timestamp }}}', + }, + }, + { + set: { + field: 'event.failure.message', + value: '{{{ _ingest.on_failure_message }}}', + }, + }, + { + set: { + field: 'event.failure.processor_type', + value: '{{{ _ingest.on_failure_processor_type }}}', + }, + }, + { + set: { + field: 'event.failure.processor_tag', + value: '{{{ _ingest.on_failure_processor_tag }}}', + }, + }, + { + set: { + field: 'event.failure.pipeline', + value: '{{{ _ingest.on_failure_pipeline }}}', + }, + }, +]; + +export const getInitialState = (model: ModelItem): MlInferenceState => { + const modelType = getModelType(model); + let targetField; + + if (modelType !== undefined) { + targetField = model.inference_config + ? `ml.inference.${model.inference_config[modelType].results_field}` + : undefined; + } + + return { + condition: undefined, + creatingPipeline: false, + error: false, + fieldMap: undefined, + ignoreFailure: false, + inferenceConfig: model.inference_config, + modelId: model.model_id, + onFailure: getDefaultOnFailureConfiguration(), + pipelineDescription: `Uses the pre-trained data frame analytics model ${model.model_id} to infer against the data that is being ingested in the pipeline`, + pipelineName: `ml-inference-${model.model_id}`, + pipelineCreated: false, + tag: undefined, + takeActionOnFailure: true, + targetField: targetField ?? '', + }; +}; diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/types.ts b/x-pack/plugins/ml/public/application/components/ml_inference/types.ts new file mode 100644 index 0000000000000..da0aef1a42154 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/types.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IngestInferenceProcessor } from '@elastic/elasticsearch/lib/api/types'; +import { ADD_INFERENCE_PIPELINE_STEPS } from './constants'; + +export type AddInferencePipelineSteps = + typeof ADD_INFERENCE_PIPELINE_STEPS[keyof typeof ADD_INFERENCE_PIPELINE_STEPS]; + +export interface MlInferenceState { + condition?: string; + creatingPipeline: boolean; + error: boolean; + fieldMap?: IngestInferenceProcessor['field_map']; + fieldMapError?: string; + ignoreFailure: boolean; + inferenceConfig: IngestInferenceProcessor['inference_config']; + inferenceConfigError?: string; + modelId: string; + onFailure?: IngestInferenceProcessor['on_failure']; + pipelineName: string; + pipelineNameError?: string; + pipelineDescription: string; + pipelineCreated: boolean; + pipelineError?: string; + tag?: string; + targetField: string; + targetFieldError?: string; + takeActionOnFailure: boolean; +} + +export interface AddInferencePipelineFormErrors { + targetField?: string; + fieldMap?: string; + inferenceConfig?: string; + pipelineName?: string; +} + +export type InferenceModelTypes = 'regression' | 'classification'; + +export interface AdditionalSettings { + condition?: string; + tag?: string; +} diff --git a/x-pack/plugins/ml/public/application/components/ml_inference/validation.ts b/x-pack/plugins/ml/public/application/components/ml_inference/validation.ts new file mode 100644 index 0000000000000..c86389607d54a --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/ml_inference/validation.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { IngestInferenceProcessor } from '@elastic/elasticsearch/lib/api/types'; +import { InferenceModelTypes } from './types'; +import type { AddInferencePipelineFormErrors } from './types'; + +const INVALID_PIPELINE_NAME_ERROR = i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.configure.invalidPipelineName', + { + defaultMessage: 'Name must only contain letters, numbers, underscores, and hyphens.', + } +); +const FIELD_REQUIRED_ERROR = i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.configure.emptyValueError', + { + defaultMessage: 'Field is required.', + } +); +const NO_EMPTY_INFERENCE_CONFIG_OBJECT = i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.configure.noEmptyInferenceConfigObjectError', + { + defaultMessage: 'Inference configuration cannot be an empty object.', + } +); +const PIPELINE_NAME_EXISTS_ERROR = i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.configure.pipelineNameExistsError', + { + defaultMessage: 'Name already used by another pipeline.', + } +); +const FIELD_MAP_REQUIRED_FIELDS_ERROR = i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.emptyValueError', + { + defaultMessage: 'Field map must include fields expected by the model.', + } +); +const INFERENCE_CONFIG_MODEL_TYPE_ERROR = i18n.translate( + 'xpack.ml.trainedModels.content.indices.pipelines.addInferencePipelineModal.steps.advanced.incorrectModelTypeError', + { + defaultMessage: 'Inference configuration inference type must match model type.', + } +); + +const VALID_PIPELINE_NAME_REGEX = /^[\w\-]+$/; +export const isValidPipelineName = (input: string): boolean => { + return input.length > 0 && VALID_PIPELINE_NAME_REGEX.test(input); +}; + +export const validateInferencePipelineConfigurationStep = ( + pipelineName: string, + pipelineNames: string[] +) => { + const errors: AddInferencePipelineFormErrors = {}; + + if (pipelineName.trim().length === 0 || pipelineName === '') { + errors.pipelineName = FIELD_REQUIRED_ERROR; + } else if (!isValidPipelineName(pipelineName)) { + errors.pipelineName = INVALID_PIPELINE_NAME_ERROR; + } + + const pipelineNameExists = pipelineNames.find((name) => name === pipelineName) !== undefined; + + if (pipelineNameExists) { + errors.pipelineName = PIPELINE_NAME_EXISTS_ERROR; + } + + return errors; +}; + +export const validateInferenceConfig = ( + inferenceConfig: IngestInferenceProcessor['inference_config'], + modelType?: InferenceModelTypes +) => { + const inferenceConfigKeys = Object.keys(inferenceConfig ?? {}); + let error; + + // If inference config has been changed, it cannot be an empty object + if (inferenceConfig && Object.keys(inferenceConfig).length === 0) { + error = NO_EMPTY_INFERENCE_CONFIG_OBJECT; + return error; + } + + // If populated, inference config must have the correct model type + if (inferenceConfig && inferenceConfigKeys.length > 0) { + if (modelType === inferenceConfigKeys[0]) { + return error; + } else { + error = INFERENCE_CONFIG_MODEL_TYPE_ERROR; + } + return error; + } + return error; +}; + +export const validateFieldMap = ( + modelInputFields: string[], + fieldMap: IngestInferenceProcessor['field_map'] +) => { + let error; + const fieldMapValues: string[] = Object.values(fieldMap?.field_map ?? {}); + + // If populated, field map must include at least some model input fields as values. + if (fieldMap && fieldMapValues.length > 0) { + if (fieldMapValues.some((v) => modelInputFields.includes(v))) { + return error; + } else { + error = FIELD_MAP_REQUIRED_FIELDS_ERROR; + } + } + + return error; +}; diff --git a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx index 09ba11f56c747..9cd0794d1b488 100644 --- a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx +++ b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx @@ -35,6 +35,7 @@ import { ModelItem } from './models_list'; export function useModelActions({ onTestAction, onModelsDeleteRequest, + onModelDeployRequest, onLoading, isLoading, fetchModels, @@ -43,6 +44,7 @@ export function useModelActions({ isLoading: boolean; onTestAction: (model: ModelItem) => void; onModelsDeleteRequest: (models: ModelItem[]) => void; + onModelDeployRequest: (model: ModelItem) => void; onLoading: (isLoading: boolean) => void; fetchModels: () => Promise; modelAndDeploymentIds: string[]; @@ -412,6 +414,54 @@ export function useModelActions({ } }, }, + { + name: (model) => { + const hasDeployments = model.state === MODEL_STATE.STARTED; + return ( + + <> + {i18n.translate('xpack.ml.trainedModels.modelsList.deployModelActionLabel', { + defaultMessage: 'Deploy model', + })} + + + ); + }, + description: i18n.translate('xpack.ml.trainedModels.modelsList.deployModelActionLabel', { + defaultMessage: 'Deploy model', + }), + 'data-test-subj': 'mlModelsTableRowDeployAction', + icon: 'continuityAbove', + type: 'icon', + isPrimary: false, + onClick: (model) => { + onModelDeployRequest(model); + }, + available: (item) => { + const isDfaTrainedModel = item.metadata?.analytics_config !== undefined; + return ( + isDfaTrainedModel && + !isBuiltInModel(item) && + !item.putModelConfig && + canManageIngestPipelines + ); + }, + enabled: (item) => { + return item.state !== MODEL_STATE.STARTED; + }, + }, { name: (model) => { const hasDeployments = model.state === MODEL_STATE.STARTED; @@ -492,6 +542,7 @@ export function useModelActions({ displayErrorToast, getUserConfirmation, onModelsDeleteRequest, + onModelDeployRequest, canDeleteTrainedModels, isBuiltInModel, onTestAction, diff --git a/x-pack/plugins/ml/public/application/model_management/models_list.tsx b/x-pack/plugins/ml/public/application/model_management/models_list.tsx index d5f15c10aa2c6..653963155bc68 100644 --- a/x-pack/plugins/ml/public/application/model_management/models_list.tsx +++ b/x-pack/plugins/ml/public/application/model_management/models_list.tsx @@ -62,6 +62,7 @@ import { useFieldFormatter } from '../contexts/kibana/use_field_formatter'; import { useRefresh } from '../routing/use_refresh'; import { SavedObjectsWarning } from '../components/saved_objects_warning'; import { TestTrainedModelFlyout } from './test_models'; +import { AddInferencePipelineFlyout } from '../components/ml_inference'; type Stats = Omit; @@ -134,6 +135,7 @@ export const ModelsList: FC = ({ const [items, setItems] = useState([]); const [selectedModels, setSelectedModels] = useState([]); const [modelsToDelete, setModelsToDelete] = useState([]); + const [modelToDeploy, setModelToDeploy] = useState(); const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( {} ); @@ -349,6 +351,7 @@ export const ModelsList: FC = ({ fetchModels: fetchModelsData, onTestAction: setModelToTest, onModelsDeleteRequest: setModelsToDelete, + onModelDeployRequest: setModelToDeploy, onLoading: setIsLoading, modelAndDeploymentIds, }); @@ -642,6 +645,12 @@ export const ModelsList: FC = ({ {modelToTest === null ? null : ( )} + {modelToDeploy !== undefined ? ( + + ) : null} ); }; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts index 0ea4b1d1fde4b..e6b9c1a5badc3 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts @@ -6,6 +6,7 @@ */ import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; import { useMemo } from 'react'; import type { HttpFetchQuery } from '@kbn/core/public'; @@ -58,7 +59,6 @@ export function trainedModelsApiProvider(httpService: HttpService) { return { /** * Fetches configuration information for a trained inference model. - * * @param modelId - Model ID, collection of Model IDs or Model ID pattern. * Fetches all In case nothing is provided. * @param params - Optional query params @@ -76,7 +76,6 @@ export function trainedModelsApiProvider(httpService: HttpService) { /** * Fetches usage information for trained inference models. - * * @param modelId - Model ID, collection of Model IDs or Model ID pattern. * Fetches all In case nothing is provided. * @param params - Optional query params @@ -93,7 +92,6 @@ export function trainedModelsApiProvider(httpService: HttpService) { /** * Fetches pipelines associated with provided models - * * @param modelId - Model ID, collection of Model IDs. */ getTrainedModelPipelines(modelId: string | string[]) { @@ -109,9 +107,31 @@ export function trainedModelsApiProvider(httpService: HttpService) { }); }, + /** + * Fetches all ingest pipelines + */ + getAllIngestPipelines() { + return httpService.http({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/ingest_pipelines`, + method: 'GET', + version: '1', + }); + }, + + /** + * Creates inference pipeline + */ + createInferencePipeline(pipelineName: string, pipeline: IngestPipeline) { + return httpService.http({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/create_inference_pipeline`, + method: 'POST', + body: JSON.stringify({ pipeline, pipelineName }), + version: '1', + }); + }, + /** * Deletes an existing trained inference model. - * * @param modelId - Model ID */ deleteTrainedModel( diff --git a/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json b/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json index 978dbf9dc94f3..90dbdbd9c121f 100644 --- a/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json +++ b/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json @@ -177,6 +177,8 @@ "DeleteTrainedModel", "SimulateIngestPipeline", "InferTrainedModelDeployment", + "CreateInferencePipeline", + "GetIngestPipelines", "Alerting", "PreviewAlert", diff --git a/x-pack/plugins/ml/server/models/model_management/models_provider.ts b/x-pack/plugins/ml/server/models/model_management/models_provider.ts index 702e8454660f6..e7cfcbe7fd50d 100644 --- a/x-pack/plugins/ml/server/models/model_management/models_provider.ts +++ b/x-pack/plugins/ml/server/models/model_management/models_provider.ts @@ -6,6 +6,11 @@ */ import type { IScopedClusterClient } from '@kbn/core/server'; +import { + IngestPipeline, + IngestSimulateDocument, + IngestSimulateRequest, +} from '@elastic/elasticsearch/lib/api/types'; import type { PipelineDefinition } from '../../../common/types/trained_models'; export type ModelService = ReturnType; @@ -64,5 +69,64 @@ export function modelsProvider(client: IScopedClusterClient) { pipelinesIds.map((id) => client.asCurrentUser.ingest.deletePipeline({ id })) ); }, + + /** + * Simulates the effect of the pipeline on given document. + * + */ + async simulatePipeline(docs: IngestSimulateDocument[], pipelineConfig: IngestPipeline) { + const simulateRequest: IngestSimulateRequest = { + docs, + pipeline: pipelineConfig, + }; + let result = {}; + try { + result = await client.asCurrentUser.ingest.simulate(simulateRequest); + } catch (error) { + if (error.statusCode === 404) { + // ES returns 404 when there are no pipelines + // Instead, we should return an empty response and a 200 + return result; + } + throw error; + } + + return result; + }, + + /** + * Creates the pipeline + * + */ + async createInferencePipeline(pipelineConfig: IngestPipeline, pipelineName: string) { + let result = {}; + + result = await client.asCurrentUser.ingest.putPipeline({ + id: pipelineName, + ...pipelineConfig, + }); + + return result; + }, + + /** + * Retrieves existing pipelines. + * + */ + async getPipelines() { + let result = {}; + try { + result = await client.asCurrentUser.ingest.getPipeline(); + } catch (error) { + if (error.statusCode === 404) { + // ES returns 404 when there are no pipelines + // Instead, we should return an empty response and a 200 + return result; + } + throw error; + } + + return result; + }, }; } diff --git a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts index e15fd108c5f5b..21b62c6f5ce42 100644 --- a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts @@ -77,3 +77,13 @@ export const deleteTrainedModelQuerySchema = schema.object({ with_pipelines: schema.maybe(schema.boolean({ defaultValue: false })), force: schema.maybe(schema.boolean({ defaultValue: false })), }); + +export const createIngestPipelineSchema = schema.object({ + pipelineName: schema.string(), + pipeline: schema.maybe( + schema.object({ + processors: schema.arrayOf(schema.any()), + description: schema.maybe(schema.string()), + }) + ), +}); diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts index cefb8a9dce0fd..c85429f41a72a 100644 --- a/x-pack/plugins/ml/server/routes/trained_models.ts +++ b/x-pack/plugins/ml/server/routes/trained_models.ts @@ -22,6 +22,7 @@ import { putTrainedModelQuerySchema, threadingParamsSchema, updateDeploymentParamsSchema, + createIngestPipelineSchema, } from './schemas/inference_schema'; import { TrainedModelConfigResponse } from '../../common/types/trained_models'; import { mlLog } from '../lib/log'; @@ -237,6 +238,78 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization) }) ); + /** + * @apiGroup TrainedModels + * + * @api {get} /internal/ml/trained_models/ingest_pipelines Get ingest pipelines + * @apiName GetIngestPipelines + * @apiDescription Retrieves pipelines + */ + router.versioned + .get({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/ingest_pipelines`, + access: 'internal', + options: { + tags: ['access:ml:canGetTrainedModels'], // TODO: update permissions + }, + }) + .addVersion( + { + version: '1', + validate: false, + }, + routeGuard.fullLicenseAPIGuard(async ({ client, request, mlClient, response }) => { + try { + const body = await modelsProvider(client).getPipelines(); + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup TrainedModels + * + * @api {post} /internal/ml/trained_models/create_inference_pipeline creates the pipeline with inference processor + * @apiName CreateInferencePipeline + * @apiDescription Creates the inference pipeline + */ + router.versioned + .post({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/create_inference_pipeline`, + access: 'internal', + options: { + tags: ['access:ml:canCreateTrainedModels'], + }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + body: createIngestPipelineSchema, + }, + }, + }, + routeGuard.fullLicenseAPIGuard(async ({ client, request, mlClient, response }) => { + try { + const { pipeline, pipelineName } = request.body; + const body = await modelsProvider(client).createInferencePipeline( + pipeline!, + pipelineName + ); + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + /** * @apiGroup TrainedModels * From 727d58f7ee841626bea492a04f04366ff97a8521 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 8 Aug 2023 14:29:25 -0400 Subject: [PATCH 2/5] skip failing test suite (#162995) --- test/functional/apps/visualize/group5/_tsvb_time_series.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/functional/apps/visualize/group5/_tsvb_time_series.ts b/test/functional/apps/visualize/group5/_tsvb_time_series.ts index 823276f0b21c8..28aa95ad24263 100644 --- a/test/functional/apps/visualize/group5/_tsvb_time_series.ts +++ b/test/functional/apps/visualize/group5/_tsvb_time_series.ts @@ -23,7 +23,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const browser = getService('browser'); const kibanaServer = getService('kibanaServer'); - describe('visual builder', function describeIndexTests() { + // Failing: See https://github.com/elastic/kibana/issues/162995 + describe.skip('visual builder', function describeIndexTests() { before(async () => { await security.testUser.setRoles([ 'kibana_admin', From 596663c3cee934de3fd5f4ff094df8af74c6bece Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 8 Aug 2023 14:30:18 -0400 Subject: [PATCH 3/5] skip failing test suite (#162672) --- x-pack/test/functional/apps/infra/hosts_view.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/infra/hosts_view.ts b/x-pack/test/functional/apps/infra/hosts_view.ts index d909e493bf2c5..f5dc470587c37 100644 --- a/x-pack/test/functional/apps/infra/hosts_view.ts +++ b/x-pack/test/functional/apps/infra/hosts_view.ts @@ -155,7 +155,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { return !!currentUrl.match(path); }); - describe('Hosts View', function () { + // Failing: See https://github.com/elastic/kibana/issues/162672 + describe.skip('Hosts View', function () { before(async () => { await Promise.all([ esArchiver.load('x-pack/test/functional/es_archives/infra/alerts'), From 7e234b1a785e653cfa92cc5f986beadfcf422e36 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Tue, 8 Aug 2023 15:23:59 -0400 Subject: [PATCH 4/5] [EventLog] change to use Data stream lifecycle instead of ILM (#163210) resolves https://github.com/elastic/kibana/issues/162886 The default continues to be 90 days for data detetention of event log documents, and the rollover is now controlled by DLM, as described in [Data stream lifecycle][]. [Data stream lifecycle]: https://www.elastic.co/guide/en/elasticsearch/reference/8.9/data-stream-lifecycle.html ## Release note Fixes the event log data stream to use Data stream lifecycle instead of Index Lifecycle Management. If you had previously customized the Kibana event log ILM policy, you should now update the lifecycle of the event log data stream itself. --- ...lerting-production-considerations.asciidoc | 10 ++-- .../server/es/cluster_client_adapter.mock.ts | 2 - .../server/es/cluster_client_adapter.test.ts | 50 ------------------- .../server/es/cluster_client_adapter.ts | 30 ----------- .../event_log/server/es/context.test.ts | 10 +--- .../event_log/server/es/documents.test.ts | 13 +---- .../plugins/event_log/server/es/documents.ts | 34 ++----------- .../plugins/event_log/server/es/init.test.ts | 44 ---------------- x-pack/plugins/event_log/server/es/init.ts | 15 +----- .../plugins/event_log/server/es/names.mock.ts | 1 - .../plugins/event_log/server/es/names.test.ts | 9 ---- x-pack/plugins/event_log/server/es/names.ts | 5 -- 12 files changed, 10 insertions(+), 213 deletions(-) diff --git a/docs/user/production-considerations/alerting-production-considerations.asciidoc b/docs/user/production-considerations/alerting-production-considerations.asciidoc index e3d343475e175..59c8a4bfa6d15 100644 --- a/docs/user/production-considerations/alerting-production-considerations.asciidoc +++ b/docs/user/production-considerations/alerting-production-considerations.asciidoc @@ -56,14 +56,10 @@ Predicting the buffer required to account for actions depends heavily on the rul experimental[] -Alerts and actions log activity in a set of "event log" indices. These indices are configured with an index lifecycle management (ILM) policy, which you can customize. The default policy rolls over the index when it reaches 50GB, or after 30 days. Indices over 90 days old are deleted. +Alerts and actions log activity in a set of "event log" data streams, one per Kibana version, named `.kibana-event-log-{VERSION}`. These data streams are configured with a lifecycle data retention of 90 days. This can be updated to other values via the standard data stream lifecycle APIs. Note that the event log data contains the data shown in the alerting pages in {kib}, so reducing the data retention period will result in less data being available to view. -The name of the index policy is `kibana-event-log-policy`. {kib} creates the index policy on startup, if it doesn't already exist. The index policy can be customized for your environment, but {kib} never modifies the index policy after creating it. - -Because {kib} uses the documents to display historic data, you should set the delete phase longer than you would like the historic data to be shown. For example, if you would like to see one month's worth of historic data, you should set the delete phase to at least one month. - -For more information on index lifecycle management, see: -{ref}/index-lifecycle-management.html[Index Lifecycle Policies]. +For more information on data stream lifecycle management, see: +{ref}/data-stream-lifecycle.html[Data stream lifecycle]. [float] [[alerting-circuit-breakers]] diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts index 6659f0b19ebeb..2a5582347db74 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts @@ -11,8 +11,6 @@ const createClusterClientMock = () => { const mock: jest.Mocked = { indexDocument: jest.fn(), indexDocuments: jest.fn(), - doesIlmPolicyExist: jest.fn(), - createIlmPolicy: jest.fn(), doesIndexTemplateExist: jest.fn(), createIndexTemplate: jest.fn(), doesDataStreamExist: jest.fn(), diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 151a65573125c..6f36af2be1f78 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -165,56 +165,6 @@ describe('buffering documents', () => { }); }); -describe('doesIlmPolicyExist', () => { - // ElasticsearchError can be a bit random in shape, we need an any here - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const notFoundError = new Error('Not found') as any; - notFoundError.statusCode = 404; - - test('should call cluster with proper arguments', async () => { - await clusterClientAdapter.doesIlmPolicyExist('foo'); - expect(clusterClient.transport.request).toHaveBeenCalledWith({ - method: 'GET', - path: '/_ilm/policy/foo', - }); - }); - - test('should return false when 404 error is returned by Elasticsearch', async () => { - clusterClient.transport.request.mockRejectedValue(notFoundError); - await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(false); - }); - - test('should throw error when error is not 404', async () => { - clusterClient.transport.request.mockRejectedValue(new Error('Fail')); - await expect( - clusterClientAdapter.doesIlmPolicyExist('foo') - ).rejects.toThrowErrorMatchingInlineSnapshot(`"error checking existance of ilm policy: Fail"`); - }); - - test('should return true when no error is thrown', async () => { - await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(true); - }); -}); - -describe('createIlmPolicy', () => { - test('should call cluster client with given policy', async () => { - clusterClient.transport.request.mockResolvedValue({ success: true }); - await clusterClientAdapter.createIlmPolicy('foo', { args: true }); - expect(clusterClient.transport.request).toHaveBeenCalledWith({ - method: 'PUT', - path: '/_ilm/policy/foo', - body: { args: true }, - }); - }); - - test('should throw error when call cluster client throws', async () => { - clusterClient.transport.request.mockRejectedValue(new Error('Fail')); - await expect( - clusterClientAdapter.createIlmPolicy('foo', { args: true }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"error creating ilm policy: Fail"`); - }); -}); - describe('doesIndexTemplateExist', () => { test('should call cluster with proper arguments', async () => { await clusterClientAdapter.doesIndexTemplateExist('foo'); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 8807e34cfedf3..27a86b839d6c9 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -178,36 +178,6 @@ export class ClusterClientAdapter { - const request = { - method: 'GET', - path: `/_ilm/policy/${policyName}`, - }; - try { - const esClient = await this.elasticsearchClientPromise; - await esClient.transport.request(request); - } catch (err) { - if (err.statusCode === 404) return false; - throw new Error(`error checking existance of ilm policy: ${err.message}`); - } - return true; - } - - public async createIlmPolicy(policyName: string, policy: Record): Promise { - this.logger.info(`Installing ILM policy ${policyName}`); - const request = { - method: 'PUT', - path: `/_ilm/policy/${policyName}`, - body: policy, - }; - try { - const esClient = await this.elasticsearchClientPromise; - await esClient.transport.request(request); - } catch (err) { - throw new Error(`error creating ilm policy: ${err.message}`); - } - } - public async doesIndexTemplateExist(name: string): Promise { try { const esClient = await this.elasticsearchClientPromise; diff --git a/x-pack/plugins/event_log/server/es/context.test.ts b/x-pack/plugins/event_log/server/es/context.test.ts index 67ea2a95151f2..681b927478d81 100644 --- a/x-pack/plugins/event_log/server/es/context.test.ts +++ b/x-pack/plugins/event_log/server/es/context.test.ts @@ -57,13 +57,12 @@ describe('createEsContext', () => { expect(esNames).toStrictEqual({ base: 'test-index', dataStream: 'test-index-event-log-1.2.3', - ilmPolicy: 'test-index-event-log-policy', indexPattern: 'test-index-event-log-*', indexTemplate: 'test-index-event-log-1.2.3-template', }); }); - test('should return exist false for esAdapter ilm policy, index template and data stream before initialize', async () => { + test('should return exist false for esAdapter index template and data stream before initialize', async () => { const context = createEsContext({ logger, indexNameRoot: 'test1', @@ -84,7 +83,7 @@ describe('createEsContext', () => { expect(doesIndexTemplateExist).toBeFalsy(); }); - test('should return exist true for esAdapter ilm policy, index template and data stream after initialize', async () => { + test('should return exist true for esAdapter index template and data stream after initialize', async () => { const context = createEsContext({ logger, indexNameRoot: 'test2', @@ -94,11 +93,6 @@ describe('createEsContext', () => { elasticsearchClient.indices.existsTemplate.mockResponse(true); context.initialize(); - const doesIlmPolicyExist = await context.esAdapter.doesIlmPolicyExist( - context.esNames.ilmPolicy - ); - expect(doesIlmPolicyExist).toBeTruthy(); - elasticsearchClient.indices.getDataStream.mockResolvedValue(GetDataStreamsResponse); const doesDataStreamExist = await context.esAdapter.doesDataStreamExist( context.esNames.dataStream diff --git a/x-pack/plugins/event_log/server/es/documents.test.ts b/x-pack/plugins/event_log/server/es/documents.test.ts index 814596f751c61..71b75ee3ca3dc 100644 --- a/x-pack/plugins/event_log/server/es/documents.test.ts +++ b/x-pack/plugins/event_log/server/es/documents.test.ts @@ -5,19 +5,9 @@ * 2.0. */ -import { getIndexTemplate, getIlmPolicy } from './documents'; +import { getIndexTemplate } from './documents'; import { getEsNames } from './names'; -describe('getIlmPolicy()', () => { - test('returns the basic structure of an ilm policy', () => { - expect(getIlmPolicy()).toMatchObject({ - policy: { - phases: {}, - }, - }); - }); -}); - describe('getIndexTemplate()', () => { const kibanaVersion = '1.2.3'; const esNames = getEsNames('XYZ', kibanaVersion); @@ -27,7 +17,6 @@ describe('getIndexTemplate()', () => { expect(indexTemplate.index_patterns).toEqual([esNames.dataStream]); expect(indexTemplate.template.settings.number_of_shards).toBeGreaterThanOrEqual(0); expect(indexTemplate.template.settings.auto_expand_replicas).toBe('0-1'); - expect(indexTemplate.template.settings['index.lifecycle.name']).toBe(esNames.ilmPolicy); expect(indexTemplate.template.mappings).toMatchObject({}); }); }); diff --git a/x-pack/plugins/event_log/server/es/documents.ts b/x-pack/plugins/event_log/server/es/documents.ts index deaa8349971f9..0f654f80ad55b 100644 --- a/x-pack/plugins/event_log/server/es/documents.ts +++ b/x-pack/plugins/event_log/server/es/documents.ts @@ -25,7 +25,9 @@ export function getIndexTemplate(esNames: EsNames) { hidden: true, number_of_shards: 1, auto_expand_replicas: '0-1', - 'index.lifecycle.name': esNames.ilmPolicy, + }, + lifecycle: { + data_retention: '90d', }, mappings, }, @@ -33,33 +35,3 @@ export function getIndexTemplate(esNames: EsNames) { return indexTemplateBody; } - -// returns the body of an ilm policy used in an ES PUT _ilm/policy call -export function getIlmPolicy() { - return { - policy: { - _meta: { - description: - 'ilm policy the Kibana event log, created initially by Kibana, but updated by the user, not Kibana', - managed: false, - }, - phases: { - hot: { - actions: { - rollover: { - max_size: '50GB', - max_age: '30d', - // max_docs: 1, // you know, for testing - }, - }, - }, - delete: { - min_age: '90d', - actions: { - delete: {}, - }, - }, - }, - }, - }; -} diff --git a/x-pack/plugins/event_log/server/es/init.test.ts b/x-pack/plugins/event_log/server/es/init.test.ts index 30e220313b26b..7c81ae80b8823 100644 --- a/x-pack/plugins/event_log/server/es/init.test.ts +++ b/x-pack/plugins/event_log/server/es/init.test.ts @@ -83,7 +83,6 @@ describe('initializeEs', () => { `error getting existing index templates - Fail` ); expect(esContext.esAdapter.setLegacyIndexTemplateToHidden).not.toHaveBeenCalled(); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should continue initialization if updating existing index templates throws an error`, async () => { @@ -124,7 +123,6 @@ describe('initializeEs', () => { expect(esContext.logger.error).toHaveBeenCalledWith( `error setting existing \"foo-bar-template\" index template to hidden - Fail` ); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should update existing index settings if any exist and are not hidden`, async () => { @@ -207,7 +205,6 @@ describe('initializeEs', () => { expect(esContext.esAdapter.getExistingIndices).toHaveBeenCalled(); expect(esContext.logger.error).toHaveBeenCalledWith(`error getting existing indices - Fail`); expect(esContext.esAdapter.setIndexToHidden).not.toHaveBeenCalled(); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should continue initialization if updating existing index settings throws an error`, async () => { @@ -251,7 +248,6 @@ describe('initializeEs', () => { expect(esContext.logger.error).toHaveBeenCalledWith( `error setting existing \"foo-bar-000001\" index to hidden - Fail` ); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should update existing index aliases if any exist and are not hidden`, async () => { @@ -300,7 +296,6 @@ describe('initializeEs', () => { `error getting existing index aliases - Fail` ); expect(esContext.esAdapter.setIndexAliasToHidden).not.toHaveBeenCalled(); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should continue initialization if updating existing index aliases throws an error`, async () => { @@ -336,23 +331,6 @@ describe('initializeEs', () => { expect(esContext.logger.error).toHaveBeenCalledWith( `error setting existing \"foo-bar\" index aliases - Fail` ); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); - }); - - test(`should create ILM policy if it doesn't exist`, async () => { - esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(false); - - await initializeEs(esContext); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); - expect(esContext.esAdapter.createIlmPolicy).toHaveBeenCalled(); - }); - - test(`shouldn't create ILM policy if it exists`, async () => { - esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true); - - await initializeEs(esContext); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); - expect(esContext.esAdapter.createIlmPolicy).not.toHaveBeenCalled(); }); test(`should create index template if it doesn't exist`, async () => { @@ -463,30 +441,10 @@ describe('retries', () => { esContext.esAdapter.getExistingLegacyIndexTemplates.mockResolvedValue({}); esContext.esAdapter.getExistingIndices.mockResolvedValue({}); esContext.esAdapter.getExistingIndexAliases.mockResolvedValue({}); - esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true); esContext.esAdapter.doesIndexTemplateExist.mockResolvedValue(true); esContext.esAdapter.doesDataStreamExist.mockResolvedValue(true); }); - test('createIlmPolicyIfNotExists with 1 retry', async () => { - esContext.esAdapter.doesIlmPolicyExist.mockRejectedValueOnce(new Error('retry 1')); - - const timeStart = performance.now(); - await initializeEs(esContext); - const timeElapsed = Math.ceil(performance.now() - timeStart); - - expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY); - - expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(2); - expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1); - - const prefix = `eventLog initialization operation failed and will be retried: createIlmPolicyIfNotExists`; - expect(esContext.logger.warn).toHaveBeenCalledTimes(1); - expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 4 more times; error: retry 1`); - }); - test('createIndexTemplateIfNotExists with 2 retries', async () => { esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2a')); esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2b')); @@ -498,7 +456,6 @@ describe('retries', () => { expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2)); expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1); expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(3); expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1); @@ -524,7 +481,6 @@ describe('retries', () => { expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2 + 4 + 8)); expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1); expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1); expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(5); expect(esContext.esAdapter.createDataStream).toHaveBeenCalledTimes(0); diff --git a/x-pack/plugins/event_log/server/es/init.ts b/x-pack/plugins/event_log/server/es/init.ts index 6eb4d5736a4a1..cf737cbf035c6 100644 --- a/x-pack/plugins/event_log/server/es/init.ts +++ b/x-pack/plugins/event_log/server/es/init.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { asyncForEach } from '@kbn/std'; import { groupBy } from 'lodash'; import pRetry, { FailedAttemptError } from 'p-retry'; -import { getIlmPolicy, getIndexTemplate } from './documents'; +import { getIndexTemplate } from './documents'; import { EsContext } from './context'; const MAX_RETRY_DELAY = 30000; @@ -33,7 +33,6 @@ async function initializeEsResources(esContext: EsContext) { // today, setExistingAssetsToHidden() never throws, but just in case ... await retry(steps.setExistingAssetsToHidden); - await retry(steps.createIlmPolicyIfNotExists); await retry(steps.createIndexTemplateIfNotExists); await retry(steps.createDataStreamIfNotExists); @@ -202,18 +201,6 @@ class EsInitializationSteps { await this.setExistingIndexAliasesToHidden(); } - async createIlmPolicyIfNotExists(): Promise { - const exists = await this.esContext.esAdapter.doesIlmPolicyExist( - this.esContext.esNames.ilmPolicy - ); - if (!exists) { - await this.esContext.esAdapter.createIlmPolicy( - this.esContext.esNames.ilmPolicy, - getIlmPolicy() - ); - } - } - async createIndexTemplateIfNotExists(): Promise { const exists = await this.esContext.esAdapter.doesIndexTemplateExist( this.esContext.esNames.indexTemplate diff --git a/x-pack/plugins/event_log/server/es/names.mock.ts b/x-pack/plugins/event_log/server/es/names.mock.ts index 138d99aa706ea..837abe9dd413b 100644 --- a/x-pack/plugins/event_log/server/es/names.mock.ts +++ b/x-pack/plugins/event_log/server/es/names.mock.ts @@ -11,7 +11,6 @@ const createNamesMock = () => { const mock: jest.Mocked = { base: '.kibana', dataStream: '.kibana-event-log-8.0.0', - ilmPolicy: 'kibana-event-log-policy', indexPattern: '.kibana-event-log-*', indexTemplate: '.kibana-event-log-8.0.0-template', }; diff --git a/x-pack/plugins/event_log/server/es/names.test.ts b/x-pack/plugins/event_log/server/es/names.test.ts index 0a05d560b9c94..63d1ad9d398a7 100644 --- a/x-pack/plugins/event_log/server/es/names.test.ts +++ b/x-pack/plugins/event_log/server/es/names.test.ts @@ -18,16 +18,7 @@ describe('getEsNames()', () => { const esNames = getEsNames(base, kibanaVersion); expect(esNames.base).toEqual(base); expect(esNames.dataStream).toEqual(`${base}-event-log-${kibanaVersion}`); - expect(esNames.ilmPolicy).toEqual(`${base}-event-log-policy`); expect(esNames.indexPattern).toEqual(`${base}-event-log-*`); expect(esNames.indexTemplate).toEqual(`${base}-event-log-${kibanaVersion}-template`); }); - - test('ilm policy name does not contain dot prefix', () => { - const base = '.XYZ'; - const kibanaVersion = '1.2.3'; - - const esNames = getEsNames(base, kibanaVersion); - expect(esNames.ilmPolicy).toEqual('XYZ-event-log-policy'); - }); }); diff --git a/x-pack/plugins/event_log/server/es/names.ts b/x-pack/plugins/event_log/server/es/names.ts index d807e53c6abbb..0e48ca911b95a 100644 --- a/x-pack/plugins/event_log/server/es/names.ts +++ b/x-pack/plugins/event_log/server/es/names.ts @@ -10,7 +10,6 @@ const EVENT_LOG_NAME_SUFFIX = `-event-log`; export interface EsNames { base: string; dataStream: string; - ilmPolicy: string; indexPattern: string; indexTemplate: string; } @@ -19,13 +18,9 @@ export function getEsNames(baseName: string, kibanaVersion: string): EsNames { const EVENT_LOG_VERSION_SUFFIX = `-${kibanaVersion.toLocaleLowerCase()}`; const eventLogName = `${baseName}${EVENT_LOG_NAME_SUFFIX}`; const eventLogNameWithVersion = `${eventLogName}${EVENT_LOG_VERSION_SUFFIX}`; - const eventLogPolicyName = `${ - baseName.startsWith('.') ? baseName.substring(1) : baseName - }${EVENT_LOG_NAME_SUFFIX}-policy`; return { base: baseName, dataStream: eventLogNameWithVersion, - ilmPolicy: `${eventLogPolicyName}`, indexPattern: `${eventLogName}-*`, indexTemplate: `${eventLogNameWithVersion}-template`, }; From b8841bcbfc81c3d9504f8f25da06d42c9ea98c80 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 8 Aug 2023 23:05:27 +0300 Subject: [PATCH 5/5] [Upgrade Assistant] Fix functional test for shards check (#163438) ## Summary Create a new functional config file that sets up elasticsearch configs to have a low disk threshold and a low number of shards per node to test for health checks and deprecations. Previously this test failed because it seems that ES takes some time to calculate the health checks hence the indicator critical issues are not showing during the testing period (now we don't have flakiness since we started the server with the indicators already in place) it also means less `before` and `after` work inside the test cases. Closes https://github.com/elastic/kibana/issues/160833 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/ftr_configs.yml | 1 + .../upgrade_assistant/deprecation_pages.ts | 64 +++++-------------- .../es_deprecation_logs_page.ts | 2 +- .../upgrade_assistant_security.ts | 1 + .../apps/upgrade_assistant/overview_page.ts | 2 +- x-pack/test/functional/config.base.js | 11 ++-- .../functional/config.upgrade_assistant.ts | 38 +++++++++++ 7 files changed, 64 insertions(+), 55 deletions(-) create mode 100644 x-pack/test/functional/config.upgrade_assistant.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 70be71f08184a..2e7f5a815c6ea 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -328,6 +328,7 @@ enabled: - x-pack/test/functional/config_security_basic.ts - x-pack/test/functional/config.ccs.ts - x-pack/test/functional/config.firefox.js + - x-pack/test/functional/config.upgrade_assistant.ts - x-pack/test/functional_cloud/config.ts - x-pack/test/kubernetes_security/basic/config.ts - x-pack/test/licensing_plugin/config.public.ts diff --git a/x-pack/test/functional/apps/upgrade_assistant/deprecation_pages.ts b/x-pack/test/functional/apps/upgrade_assistant/deprecation_pages.ts index 1eb4752026b75..19b4fad72d272 100644 --- a/x-pack/test/functional/apps/upgrade_assistant/deprecation_pages.ts +++ b/x-pack/test/functional/apps/upgrade_assistant/deprecation_pages.ts @@ -6,7 +6,6 @@ */ import expect from '@kbn/expect'; -import { setTimeout } from 'timers/promises'; import { FtrProviderContext } from '../../ftr_provider_context'; export default function upgradeAssistantFunctionalTests({ @@ -20,50 +19,27 @@ export default function upgradeAssistantFunctionalTests({ const security = getService('security'); const log = getService('log'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/160833 - describe.skip('Deprecation pages', function () { - this.tags('skipFirefox'); + describe('Deprecation pages', function () { + this.tags(['skipFirefox', 'upgradeAssistant']); before(async () => { await security.testUser.setRoles(['superuser']); - - // Cluster readiness checks try { - // Trigger "Total shards" ES Upgrade readiness check + /** + * Trigger "Total shards" ES Upgrade readiness check + * the number of shards in the test cluster is 25-27 + * so 5 max shards per node should trigger this check + * on both local and CI environments. + */ await es.cluster.putSettings({ body: { persistent: { cluster: { - max_shards_per_node: '9', + max_shards_per_node: 5, }, }, }, }); - - // Trigger "Low disk watermark" ES Upgrade readiness check - await es.cluster.putSettings({ - body: { - persistent: { - cluster: { - // push allocation changes to nodes quickly during tests - info: { - update: { interval: '10s' }, - }, - routing: { - allocation: { - disk: { - threshold_enabled: true, - watermark: { low: '30%' }, - }, - }, - }, - }, - }, - }, - }); - - // Wait for the cluster settings to be reflected to the ES nodes - await setTimeout(12000); } catch (e) { log.debug('[Setup error] Error updating cluster settings'); throw e; @@ -76,18 +52,8 @@ export default function upgradeAssistantFunctionalTests({ body: { persistent: { cluster: { - info: { - update: { interval: null }, - }, - max_shards_per_node: null, - routing: { - allocation: { - disk: { - threshold_enabled: false, - watermark: { low: null }, - }, - }, - }, + // initial cluster setting from x-pack/test/functional/config.upgrade_assistant.js + max_shards_per_node: 27, }, }, }, @@ -96,7 +62,6 @@ export default function upgradeAssistantFunctionalTests({ log.debug('[Cleanup error] Error reseting cluster settings'); throw e; } - await security.testUser.restoreDefaults(); }); @@ -111,10 +76,13 @@ export default function upgradeAssistantFunctionalTests({ it('renders the Elasticsearch upgrade readiness deprecations', async () => { const deprecationMessages = await testSubjects.getVisibleTextAll('defaultTableCell-message'); + const healthIndicatorsCriticalMessages = await testSubjects.getVisibleTextAll( + 'healthIndicatorTableCell-message' + ); expect(deprecationMessages).to.contain('Disk usage exceeds low watermark'); - expect(deprecationMessages).to.contain( - 'The cluster has too many shards to be able to upgrade' + expect(healthIndicatorsCriticalMessages).to.contain( + 'Elasticsearch is about to reach the maximum number of shards it can host, based on your current settings.' ); }); diff --git a/x-pack/test/functional/apps/upgrade_assistant/es_deprecation_logs_page.ts b/x-pack/test/functional/apps/upgrade_assistant/es_deprecation_logs_page.ts index 2ba3531486fff..647d692299075 100644 --- a/x-pack/test/functional/apps/upgrade_assistant/es_deprecation_logs_page.ts +++ b/x-pack/test/functional/apps/upgrade_assistant/es_deprecation_logs_page.ts @@ -17,7 +17,7 @@ export default function upgradeAssistantESDeprecationLogsPageFunctionalTests({ const es = getService('es'); describe('ES deprecation logs page', function () { - this.tags('skipFirefox'); + this.tags(['skipFirefox', 'upgradeAssistant']); before(async () => { await security.testUser.setRoles(['superuser']); diff --git a/x-pack/test/functional/apps/upgrade_assistant/feature_controls/upgrade_assistant_security.ts b/x-pack/test/functional/apps/upgrade_assistant/feature_controls/upgrade_assistant_security.ts index 755f69cb43c20..a7c883733ea13 100644 --- a/x-pack/test/functional/apps/upgrade_assistant/feature_controls/upgrade_assistant_security.ts +++ b/x-pack/test/functional/apps/upgrade_assistant/feature_controls/upgrade_assistant_security.ts @@ -15,6 +15,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const managementMenu = getService('managementMenu'); describe('security', function () { + this.tags('upgradeAssistant'); before(async () => { await PageObjects.common.navigateToApp('home'); }); diff --git a/x-pack/test/functional/apps/upgrade_assistant/overview_page.ts b/x-pack/test/functional/apps/upgrade_assistant/overview_page.ts index 6629ec6b9cd67..10493d1df8117 100644 --- a/x-pack/test/functional/apps/upgrade_assistant/overview_page.ts +++ b/x-pack/test/functional/apps/upgrade_assistant/overview_page.ts @@ -16,7 +16,7 @@ export default function upgradeAssistantOverviewPageFunctionalTests({ const testSubjects = getService('testSubjects'); describe('Overview Page', function () { - this.tags('skipFirefox'); + this.tags(['skipFirefox', 'upgradeAssistant']); before(async () => { await security.testUser.setRoles(['superuser']); diff --git a/x-pack/test/functional/config.base.js b/x-pack/test/functional/config.base.js index e4427d66e534d..36dc0c6d84c3c 100644 --- a/x-pack/test/functional/config.base.js +++ b/x-pack/test/functional/config.base.js @@ -35,11 +35,7 @@ export default async function ({ readConfigFile }) { esTestCluster: { license: 'trial', from: 'snapshot', - serverArgs: [ - 'path.repo=/tmp/', - 'xpack.security.authc.api_key.enabled=true', - 'cluster.routing.allocation.disk.threshold_enabled=true', // make sure disk thresholds are enabled for UA cluster testing - ], + serverArgs: ['path.repo=/tmp/', 'xpack.security.authc.api_key.enabled=true'], }, kbnTestServer: { @@ -183,6 +179,11 @@ export default async function ({ readConfigFile }) { }, }, + suiteTags: { + ...kibanaCommonConfig.get('suiteTags'), + exclude: [...kibanaCommonConfig.get('suiteTags').exclude, 'upgradeAssistant'], + }, + // choose where screenshots should be saved screenshots: { directory: resolve(__dirname, 'screenshots'), diff --git a/x-pack/test/functional/config.upgrade_assistant.ts b/x-pack/test/functional/config.upgrade_assistant.ts new file mode 100644 index 0000000000000..a9e0a447a2961 --- /dev/null +++ b/x-pack/test/functional/config.upgrade_assistant.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('./config.base.js')); + + return { + ...functionalConfig.getAll(), + + testFiles: [require.resolve('./apps/upgrade_assistant')], + + junit: { + reportName: 'Chrome X-Pack UI Upgrade Assistant Functional Tests', + }, + + suiteTags: { + include: ['upgradeAssistant'], + }, + + esTestCluster: { + ...functionalConfig.get('esTestCluster'), + serverArgs: [ + 'path.repo=/tmp/', + 'xpack.security.authc.api_key.enabled=true', + 'cluster.routing.allocation.disk.threshold_enabled=true', // make sure disk thresholds are enabled for UA cluster testing + 'cluster.routing.allocation.disk.watermark.low=30%', + 'cluster.info.update.interval=10s', + 'cluster.max_shards_per_node=27', + ], + }, + }; +}