From b406115eb1dccf63c9d7b3cd5b690e71ba1ec86f Mon Sep 17 00:00:00 2001 From: Sonia Sanz Vivas Date: Wed, 27 Nov 2024 07:42:30 +0100 Subject: [PATCH] Handle both JSON and XJSON in processor editor (#200692) Closes [#175753](https://github.com/elastic/kibana/issues/175753) ## Summary When a user creates a pipeline with a processor that has an JSON editor field [Foreach, Grok, Inference, Redact and Script], our editor convert the input to XJSON. This can be confused to the user, who see their input modified without understanding the reason. For that, this PR creates a new editor that handles both XJSON and JSON format and does not changes the content that the user introduced while editing. Once the pipeline is saved, it will always retrieve the parsed data. I've created a whole new editor instead of modifying `XJsonEditor` because the original one is still in use in the `Custom processor`. This is because I've created [a list](https://github.com/SoniaSanzV/kibana/blob/d7d5ecafa7dbae96fe52c2e37394520b6353bd92/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts#L151-L157) of the processor fields that needs to be serialized in `convertProcessorInternalToProcessor`. Since we don't know the name of the custom processor, we can't apply this workflow. I'm not super happy with the idea of adding manually the names of the processors to a list. I would like to have something more agnostic but I haven't come up with a solution that allows me to do that without breaking other fields. So, any suggestions are super welcome! ### How to test Create one of the following processors: Foreach, Grok, Inference, Redact or Script. In the JSON field, add a JSON with both with and without escaped strings. While you are editing the processor before saving the pipeline, you should see the processor as you typed it. In the `Show Request` flyout, you should see the properly parsed JSON. The pipeline should save with the correct parsed values and, once saved, if you click update, you only will get parsed values. ### Demo https://github.com/user-attachments/assets/1f9681df-2fb4-4ed5-ac30-03f2937abfe9 --- .../__jest__/processors/foreach.test.tsx | 117 ++++++++++++++++++ .../__jest__/processors/grok.test.ts | 37 ++++++ .../__jest__/processors/inference.test.tsx | 111 +++++++++++++++++ .../__jest__/processors/processor.helpers.tsx | 8 +- .../__jest__/processors/redact.test.tsx | 36 ++++++ .../__jest__/processors/script.test.tsx | 104 ++++++++++++++++ .../field_components/xjson_editor.tsx | 13 +- .../processor_form.container.tsx | 2 +- .../processor_form/processors/custom.tsx | 20 +-- .../processor_form/processors/foreach.tsx | 11 +- .../processor_form/processors/grok.tsx | 14 ++- .../processor_form/processors/inference.tsx | 28 +++-- .../processor_form/processors/redact.tsx | 12 +- .../processor_form/processors/script.tsx | 19 ++- .../processor_form/processors/shared.test.ts | 46 ++++++- .../processor_form/processors/shared.ts | 58 ++++++++- .../components/pipeline_editor/serialize.ts | 4 +- .../components/pipeline_editor/utils.test.ts | 47 ++++++- .../components/pipeline_editor/utils.ts | 51 ++++++++ 19 files changed, 675 insertions(+), 63 deletions(-) create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx new file mode 100644 index 000000000000..e4ca33fbffad --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue, setupEnvironment } from './processor.helpers'; + +const FOREACH_TYPE = 'foreach'; + +describe('Processor: Foreach', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + const { httpSetup } = setupEnvironment(); + + beforeAll(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + // disable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = true; + }); + + afterAll(() => { + jest.useRealTimers(); + // enable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = false; + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup(httpSetup, { + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + + const { component, actions } = testBed; + + component.update(); + + // Open flyout to add new processor + actions.addProcessor(); + // Add type (the other fields are not visible until a type is selected) + await actions.addProcessorType(FOREACH_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is a required parameter + expect(form.getErrorsMessages()).toEqual(['A field value is required.']); + }); + + test('saves with default parameter values', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_foreach_processor'); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, FOREACH_TYPE); + + expect(processors[0][FOREACH_TYPE]).toEqual({ + field: 'test_foreach_processor', + }); + }); + + test('accepts processor definitions that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + form, + find, + component, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_foreach_processor'); + + await act(async () => { + find('processorField').simulate('change', { + jsonContent: '{"def_1":"""aaa"bbb""", "def_2":"aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, FOREACH_TYPE); + + expect(processors[0][FOREACH_TYPE]).toEqual({ + field: 'test_foreach_processor', + // eslint-disable-next-line prettier/prettier + processor: { def_1: 'aaa\"bbb', def_2: 'aaa(bbb' }, + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts index 3f98b95ba8b6..84e494849569 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts @@ -126,4 +126,41 @@ describe('Processor: Grok', () => { expect(processors[0][GROK_TYPE].patterns).toEqual([escapedValue]); }); + + test('accepts pattern definitions that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + form, + find, + component, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_grok_processor'); + + // Add pattern 1 + form.setInputValue('droppableList.input-0', 'pattern1'); + + await act(async () => { + find('patternDefinitionsField').simulate('change', { + jsonContent: '{"pattern_1":"""aaa"bbb""", "pattern_2":"aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, GROK_TYPE); + + expect(processors[0][GROK_TYPE]).toEqual({ + field: 'test_grok_processor', + patterns: ['pattern1'], + // eslint-disable-next-line prettier/prettier + pattern_definitions: { pattern_1: 'aaa\"bbb', pattern_2: 'aaa(bbb' }, + }); + }); }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx new file mode 100644 index 000000000000..5a07fc63d087 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue, setupEnvironment } from './processor.helpers'; + +const INFERENCE_TYPE = 'inference'; + +describe('Processor: Script', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + const { httpSetup } = setupEnvironment(); + + beforeAll(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + // disable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = true; + }); + + afterAll(() => { + jest.useRealTimers(); + // enable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = false; + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup(httpSetup, { + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + + const { component, actions } = testBed; + + component.update(); + + // Open flyout to add new processor + actions.addProcessor(); + // Add type (the other fields are not visible until a type is selected) + await actions.addProcessorType(INFERENCE_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is a required parameter + expect(form.getErrorsMessages()).toEqual([ + 'A deployment, an inference, or a model ID value is required.', + ]); + }); + + test('accepts inference config and field maps that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + find, + form, + component, + } = testBed; + + form.setInputValue('inferenceModelId.input', 'test_inference_processor'); + + await act(async () => { + find('inferenceConfig').simulate('change', { + jsonContent: '{"inf_conf_1":"""aaa"bbb""", "inf_conf_2": "aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + await act(async () => { + find('fieldMap').simulate('change', { + jsonContent: '{"field_map_1":"""aaa"bbb""", "field_map_2": "aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, INFERENCE_TYPE); + + expect(processors[0][INFERENCE_TYPE]).toEqual({ + model_id: 'test_inference_processor', + // eslint-disable-next-line prettier/prettier + inference_config: { inf_conf_1: 'aaa\"bbb', inf_conf_2: 'aaa(bbb' }, + // eslint-disable-next-line prettier/prettier + field_map: { field_map_1: 'aaa\"bbb', field_map_2: 'aaa(bbb' }, + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx index 0f3ea833a311..3eeff809999b 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx @@ -199,4 +199,10 @@ type TestSubject = | 'ignoreMissingPipelineSwitch.input' | 'destinationField.input' | 'datasetField.input' - | 'namespaceField.input'; + | 'namespaceField.input' + | 'processorField' + | 'paramsField' + | 'scriptSource' + | 'inferenceModelId.input' + | 'inferenceConfig' + | 'fieldMap'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx index d34e4d1476bb..eda4bac8266d 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx @@ -138,4 +138,40 @@ describe('Processor: Redact', () => { pattern_definitions: { GITHUB_NAME: '@%{USERNAME}' }, }); }); + test('accepts pattern definitions that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + component, + find, + form, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_redact_processor'); + + // Add one pattern to the list + form.setInputValue('droppableList.input-0', 'pattern1'); + + await act(async () => { + find('patternDefinitionsField').simulate('change', { + jsonContent: '{"pattern_1":"""aaa"bbb""", "pattern_2":"aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, REDACT_TYPE); + + expect(processors[0][REDACT_TYPE]).toEqual({ + field: 'test_redact_processor', + patterns: ['pattern1'], + + pattern_definitions: { pattern_1: 'aaa"bbb', pattern_2: 'aaa(bbb' }, + }); + }); }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx new file mode 100644 index 000000000000..428c87ea6a7e --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue, setupEnvironment } from './processor.helpers'; + +const SCRIPT_TYPE = 'script'; + +describe('Processor: Script', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + const { httpSetup } = setupEnvironment(); + + beforeAll(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + // disable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = true; + }); + + afterAll(() => { + jest.useRealTimers(); + // enable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = false; + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup(httpSetup, { + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + + const { component, actions } = testBed; + + component.update(); + + // Open flyout to add new processor + actions.addProcessor(); + // Add type (the other fields are not visible until a type is selected) + await actions.addProcessorType(SCRIPT_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is a required parameter + expect(form.getErrorsMessages()).toEqual(['A value is required.']); + }); + + test('accepts params that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + find, + component, + } = testBed; + + await act(async () => { + find('scriptSource').simulate('change', { + jsonContent: 'ctx._source[params.sum_field]', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + await act(async () => { + find('paramsField').simulate('change', { + jsonContent: '{"sum_field":"""aaa"bbb"""}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, SCRIPT_TYPE); + + expect(processors[0][SCRIPT_TYPE]).toEqual({ + source: 'ctx._source[params.sum_field]', + // eslint-disable-next-line prettier/prettier + params: { sum_field: 'aaa\"bbb' }, + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx index 5134df09ac93..b5f98c260661 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx @@ -7,9 +7,7 @@ import { XJsonLang } from '@kbn/monaco'; import React, { FunctionComponent, useCallback } from 'react'; -import { FieldHook, XJson } from '../../../../../../shared_imports'; - -const { useXJsonMode } = XJson; +import { FieldHook } from '../../../../../../shared_imports'; import { TextEditor } from './text_editor'; @@ -25,20 +23,17 @@ const defaultEditorOptions = { export const XJsonEditor: FunctionComponent = ({ field, editorProps }) => { const { value, setValue } = field; - const { xJson, setXJson, convertToJson } = useXJsonMode(value); - const onChange = useCallback( (s: any) => { - setXJson(s); - setValue(convertToJson(s)); + setValue(s); }, - [setValue, setXJson, convertToJson] + [setValue] ); return ( = ({ type: formState.type, fields: formState.customOptions ? { - ...formState.customOptions, + customOptions: formState.customOptions, } : { ...formState.fields, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx index 30cc54439f26..7aa55418552d 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx @@ -15,30 +15,20 @@ import { UseField, } from '../../../../../../shared_imports'; -const { emptyField, isJsonField } = fieldValidators; +const { emptyField } = fieldValidators; import { XJsonEditor } from '../field_components'; import { Fields } from '../processor_form.container'; -import { EDITOR_PX_HEIGHT } from './shared'; +import { EDITOR_PX_HEIGHT, from, isXJsonField, to } from './shared'; const customConfig: FieldConfig = { type: FIELD_TYPES.TEXT, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.customForm.optionsFieldLabel', { defaultMessage: 'Configuration', }), - serializer: (value: string) => { - try { - return JSON.parse(value); - } catch (error) { - // swallow error and return non-parsed value; - return value; - } - }, + serializer: from.optionalXJson, deserializer: (value: any) => { - if (value === '') { - return '{\n\n}'; - } - return JSON.stringify(value, null, 2); + return to.xJsonString(value.customOptions ? value.customOptions : value); }, validations: [ { @@ -52,7 +42,7 @@ const customConfig: FieldConfig = { ), }, { - validator: isJsonField( + validator: isXJsonField( i18n.translate('xpack.ingestPipelines.pipelineEditor.customForm.invalidJsonError', { defaultMessage: 'The input is not valid.', }) diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx index 55c3a3ac1115..2bcf2847ad77 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx @@ -13,16 +13,16 @@ import { FIELD_TYPES, fieldValidators, UseField } from '../../../../../../shared import { XJsonEditor } from '../field_components'; import { FieldNameField } from './common_fields/field_name_field'; -import { FieldsConfig, to, EDITOR_PX_HEIGHT } from './shared'; +import { FieldsConfig, to, EDITOR_PX_HEIGHT, from, isXJsonField } from './shared'; -const { emptyField, isJsonField } = fieldValidators; +const { emptyField } = fieldValidators; const fieldsConfig: FieldsConfig = { /* Required fields config */ processor: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: JSON.parse, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.foreachForm.processorFieldLabel', { defaultMessage: 'Processor', }), @@ -41,7 +41,7 @@ const fieldsConfig: FieldsConfig = { ), }, { - validator: isJsonField( + validator: isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.foreachForm.processorInvalidJsonError', { @@ -68,6 +68,7 @@ export const Foreach: FunctionComponent = () => { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'processorField', height: EDITOR_PX_HEIGHT.medium, 'aria-label': i18n.translate( 'xpack.ingestPipelines.pipelineEditor.foreachForm.optionsFieldAriaLabel', diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx index ae56d3b30a62..f07fb32aa64b 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx @@ -18,13 +18,13 @@ import { ArrayItem, } from '../../../../../../shared_imports'; -import { XJsonEditor, DragAndDropTextList } from '../field_components'; +import { DragAndDropTextList, XJsonEditor } from '../field_components'; import { FieldNameField } from './common_fields/field_name_field'; import { IgnoreMissingField } from './common_fields/ignore_missing_field'; -import { FieldsConfig, to, from, EDITOR_PX_HEIGHT } from './shared'; +import { FieldsConfig, to, from, EDITOR_PX_HEIGHT, isXJsonField } from './shared'; -const { isJsonField, emptyField } = fieldValidators; +const { emptyField } = fieldValidators; const i18nTexts = { addPatternLabel: i18n.translate( @@ -70,8 +70,8 @@ const fieldsConfig: FieldsConfig = { /* Optional field configs */ pattern_definitions: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: from.optionalJson, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.grokForm.patternDefinitionsLabel', { defaultMessage: 'Pattern definitions (optional)', }), @@ -84,7 +84,7 @@ const fieldsConfig: FieldsConfig = { ), validations: [ { - validator: isJsonField( + validator: isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.grokForm.patternsDefinitionsInvalidJSONError', { defaultMessage: 'Invalid JSON' } @@ -153,6 +153,7 @@ export const Grok: FunctionComponent = () => { config={fieldsConfig.pattern_definitions} componentProps={{ editorProps: { + 'data-test-subj': 'patternDefinitionsField', height: EDITOR_PX_HEIGHT.medium, 'aria-label': i18n.translate( 'xpack.ingestPipelines.pipelineEditor.grokForm.patternDefinitionsAriaLabel', @@ -163,6 +164,7 @@ export const Grok: FunctionComponent = () => { }, }} path="fields.pattern_definitions" + data-test-subj="patternDefinitions" /> { const documentationDocsLink = services.documentation.getDocumentationUrl(); return ( <> - + { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'fieldMap', height: EDITOR_PX_HEIGHT.medium, options: { minimap: { enabled: false } }, }, @@ -173,6 +178,7 @@ export const Inference: FunctionComponent = () => { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'inferenceConfig', height: EDITOR_PX_HEIGHT.medium, options: { minimap: { enabled: false } }, }, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx index 743a5b6b7275..37e488996dbe 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx @@ -19,13 +19,13 @@ import { ValidationFunc, } from '../../../../../../shared_imports'; -import { XJsonEditor, InputList } from '../field_components'; +import { InputList, XJsonEditor } from '../field_components'; import { FieldNameField } from './common_fields/field_name_field'; import { IgnoreMissingField } from './common_fields/ignore_missing_field'; -import { FieldsConfig, to, from, EDITOR_PX_HEIGHT } from './shared'; +import { FieldsConfig, to, from, EDITOR_PX_HEIGHT, isXJsonField } from './shared'; -const { isJsonField, emptyField } = fieldValidators; +const { emptyField } = fieldValidators; const i18nTexts = { addPatternLabel: i18n.translate( @@ -68,8 +68,8 @@ const fieldsConfig: FieldsConfig = { /* Optional field configs */ pattern_definitions: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: from.optionalJson, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.redactForm.patternDefinitionsLabel', { @@ -85,7 +85,7 @@ const fieldsConfig: FieldsConfig = { ), validations: [ { - validator: isJsonField( + validator: isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.redactForm.patternsDefinitionsInvalidJSONError', { defaultMessage: 'Invalid JSON' } diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx index 18da4097eaf2..bdbf90971621 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx @@ -21,9 +21,16 @@ import { import { XJsonEditor, TextEditor } from '../field_components'; -import { FieldsConfig, to, from, FormFieldsComponent, EDITOR_PX_HEIGHT } from './shared'; +import { + FieldsConfig, + to, + from, + FormFieldsComponent, + EDITOR_PX_HEIGHT, + isXJsonField, +} from './shared'; -const { isJsonField, emptyField } = fieldValidators; +const { emptyField } = fieldValidators; const fieldsConfig: FieldsConfig = { /* Required fields config */ @@ -98,8 +105,8 @@ const fieldsConfig: FieldsConfig = { params: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: from.optionalJson, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.scriptForm.paramsFieldLabel', { defaultMessage: 'Parameters', }), @@ -113,7 +120,7 @@ const fieldsConfig: FieldsConfig = { { validator: (value) => { if (value.value) { - return isJsonField( + return isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.scriptForm.processorInvalidJsonError', { @@ -166,6 +173,7 @@ export const Script: FormFieldsComponent = ({ initialFieldValues }) => { component={TextEditor} componentProps={{ editorProps: { + 'data-test-subj': 'scriptSource', languageId: scriptLanguage, suggestionProvider: scriptLanguage === PainlessLang.ID ? suggestionProvider : undefined, @@ -191,6 +199,7 @@ export const Script: FormFieldsComponent = ({ initialFieldValues }) => { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'paramsField', height: EDITOR_PX_HEIGHT.medium, 'aria-label': i18n.translate( 'xpack.ingestPipelines.pipelineEditor.scriptForm.paramsFieldAriaLabel', diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts index 4b01f22a9383..a3b77293c66f 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { from, to } from './shared'; +import { ValidationFuncArg } from '@kbn/console-plugin/public/shared_imports'; +import { from, isXJsonField, to } from './shared'; describe('shared', () => { describe('deserialization helpers', () => { @@ -28,6 +29,21 @@ describe('shared', () => { '%{clientip} %{ident} %{auth} [%{@timestamp}] \\"%{verb} %{request} HTTP/%{httpversion}\\" %{status} %{size}' ); }); + test('to.xJsonString', () => { + const input1 = ''; + expect(to.xJsonString(input1)).toBe('{}'); + + // eslint-disable-next-line prettier/prettier + const input2 = '{"ISSUE": "aaa\"bbb","ISSUE2": "aaa\\(bbb","ISSUE3": """aaa\"bbb"""}'; + expect(to.xJsonString(input2)).toBe( + // eslint-disable-next-line prettier/prettier + '{"ISSUE": "aaa\"bbb","ISSUE2": "aaa\\(bbb","ISSUE3": """aaa\"bbb"""}' + ); + + // eslint-disable-next-line prettier/prettier + const input3 = { ISSUE: "aaa\"bbb", ISSUE2: "aaa\\(bbb" }; + expect(to.xJsonString(input3)).toBe(JSON.stringify(input3, null, 2)); + }); }); describe('serialization helpers', () => { @@ -49,5 +65,33 @@ describe('shared', () => { `%{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size}` ); }); + test('from.optionalXJson', () => { + const input1 = ''; + expect(from.optionalXJson(input1)).toBe(undefined); + + const input2 = '{}'; + expect(from.optionalXJson(input2)).toBe(undefined); + + const input3 = '{"ISSUE": "aaa","ISSUE2": "bbb"}'; + expect(from.optionalXJson(input3)).toBe(input3); + }); + }); + describe('validators', () => { + test('isXJsonField', () => { + const message = 'test error message'; + const code = 'ERR_JSON_FORMAT'; + + const validate = isXJsonField(message); + const validator = (value: unknown) => validate({ value } as ValidationFuncArg); + + // Valid JSON + const input1 = '{"ISSUE": """aaa"bbb""", "ISSUE2": """aaa\bbb"""}'; + expect(validator(input1)).toBeUndefined(); + + // Invalid JSON + // eslint-disable-next-line prettier/prettier + const input2 = '{"ISSUE": """"aaa\"bbb""'; + expect(validator(input2)).toMatchObject({ message, code }); + }); }); }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts index a14944a33a8c..f6f7f2b105de 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts @@ -10,9 +10,11 @@ import * as rt from 'io-ts'; import { i18n } from '@kbn/i18n'; import { isRight } from 'fp-ts/lib/Either'; +import { ERROR_CODE } from '@kbn/es-ui-shared-plugin/static/forms/helpers/field_validators/types'; import { FieldConfig, ValidationFunc, fieldValidators } from '../../../../../../shared_imports'; +import { collapseEscapedStrings } from '../../../utils'; -const { emptyField } = fieldValidators; +const { emptyField, isJsonField } = fieldValidators; export const arrayOfStrings = rt.array(rt.string); @@ -21,6 +23,35 @@ export function isArrayOfStrings(v: unknown): v is string[] { return isRight(res); } +/** + * Format a XJson string input as parsed JSON. Replaces the invalid characters + * with a placeholder, parses the new string in a JSON format with the expected + * indentantion and then replaces the placeholders with the original values. + */ +const formatXJsonString = (input: string) => { + let placeholder = 'PLACEHOLDER'; + const INVALID_STRING_REGEX = /"""(.*?)"""/gs; + while (input.includes(placeholder)) { + placeholder += '_'; + } + const modifiedInput = input.replace(INVALID_STRING_REGEX, () => `"${placeholder}"`); + + let jsonObject; + try { + jsonObject = JSON.parse(modifiedInput); + } catch (error) { + return input; + } + let formattedJsonString = JSON.stringify(jsonObject, null, 2); + const invalidStrings = input.match(INVALID_STRING_REGEX); + if (invalidStrings) { + invalidStrings.forEach((invalidString) => { + formattedJsonString = formattedJsonString.replace(`"${placeholder}"`, invalidString); + }); + } + return formattedJsonString; +}; + /** * Shared deserializer functions. * @@ -50,6 +81,15 @@ export const to = { } return v; }, + xJsonString: (v: unknown) => { + if (!v) { + return '{}'; + } + if (typeof v === 'string') { + return formatXJsonString(v); + } + return JSON.stringify(v, null, 2); + }, }; /** @@ -98,6 +138,12 @@ export const from = { } } }, + optionalXJson: (v: string) => { + if (v && v !== '{}') { + return v; + } + return undefined; + }, }; const isJSONString = (v: string) => { @@ -120,6 +166,16 @@ export const isJSONStringValidator: ValidationFunc = ({ value }) => { } }; +export const isXJsonField = + (message: string, { allowEmptyString = false }: { allowEmptyString?: boolean } = {}) => + (...args: Parameters): ReturnType> => { + const [{ value, ...rest }] = args; + return isJsonField(message, { allowEmptyString })({ + ...rest, + value: collapseEscapedStrings(value as string), + }); + }; + /** * Similar to the emptyField validator but we accept whitespace characters. */ diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts index e0a8fb49d5d0..3edc4ad02d6a 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts @@ -8,6 +8,7 @@ import { Processor } from '../../../../common/types'; import { ProcessorInternal } from './types'; +import { convertProccesorsToJson } from './utils'; interface SerializeArgs { /** @@ -33,9 +34,10 @@ const convertProcessorInternalToProcessor = ( copyIdToTag?: boolean ): Processor => { const { options, onFailure, type, id } = processor; + const outProcessor = { [type]: { - ...options, + ...convertProccesorsToJson(options), }, }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts index 6e367a83bf8d..8b75afd98578 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts @@ -5,7 +5,13 @@ * 2.0. */ -import { getValue, setValue, hasTemplateSnippet } from './utils'; +import { + getValue, + setValue, + hasTemplateSnippet, + collapseEscapedStrings, + convertProccesorsToJson, +} from './utils'; describe('get and set values', () => { const testObject = Object.freeze([{ onFailure: [{ onFailure: 1 }] }]); @@ -51,3 +57,42 @@ describe('template snippets', () => { expect(hasTemplateSnippet('{{{hello.world}}}')).toBe(true); }); }); + +describe('collapse escaped strings', () => { + it('returns escaped literal strings', () => { + expect(collapseEscapedStrings('{"1": """aaa\bbb""", "2": """ccc"""}')).toBe( + '{"1": "aaa\\bbb", "2": "ccc"}' + ); + }); +}); + +describe('convert processors to json', () => { + it('returns converted processors', () => { + const obj = { + field1: 'mustNotChange', + field2: 123, + field3: '{1: "mustNotChange"}', + pattern_definitions: '{"1": """aaa"bbb"""}', + processor: '{"1": """aaa"bbb"""}', + inference_config: '{"1": """aaa"bbb"""}', + field_map: '{"1": """aaa"bbb"""}', + customOptions: '{"customProcessor": """aaa"bbb"""}', + }; + + expect(convertProccesorsToJson(obj)).toEqual({ + field1: 'mustNotChange', + field2: 123, + field3: '{1: "mustNotChange"}', + // eslint-disable-next-line prettier/prettier + pattern_definitions: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + processor: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + inference_config: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + field_map: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + customProcessor: "aaa\"bbb" + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts index d7e6fc4a4d9a..533f9621bbec 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts @@ -119,3 +119,54 @@ export const hasTemplateSnippet = (str: string = '') => { // * And followed by }}} return /{{{.+}}}/.test(str); }; + +const escapeLiteralStrings = (data: string): string[] => { + const splitData = data.split(`"""`); + for (let i = 1; i < splitData.length - 1; i += 2) { + splitData[i] = JSON.stringify(splitData[i]); + } + return splitData; +}; + +const convertProcessorValueToJson = (data: string): any => { + if (!data) { + return undefined; + } + + try { + const escapedData = escapeLiteralStrings(data); + return JSON.parse(escapedData.join('')); + } catch (error) { + return data; + } +}; + +export const collapseEscapedStrings = (data: string): string => { + if (data) { + return escapeLiteralStrings(data).join(''); + } + return data; +}; + +const fieldToConvertToJson = [ + 'inference_config', + 'field_map', + 'params', + 'pattern_definitions', + 'processor', +]; + +export const convertProccesorsToJson = (obj: { [key: string]: any }): { [key: string]: any } => { + return Object.fromEntries( + Object.entries(obj).flatMap(([key, value]) => { + if (key === 'customOptions') { + const convertedValue = convertProcessorValueToJson(value); + return Object.entries(convertedValue); + } else if (fieldToConvertToJson.includes(key)) { + return [[key, convertProcessorValueToJson(value)]]; + } else { + return [[key, value]]; + } + }) + ); +};