From 8839894646451144a9534157c08cde18727aa240 Mon Sep 17 00:00:00 2001 From: Sonia Sanz Vivas Date: Wed, 27 Nov 2024 07:42:30 +0100 Subject: [PATCH] Handle both JSON and XJSON in processor editor (#200692) Closes [#175753](https://github.com/elastic/kibana/issues/175753) ## Summary When a user creates a pipeline with a processor that has an JSON editor field [Foreach, Grok, Inference, Redact and Script], our editor convert the input to XJSON. This can be confused to the user, who see their input modified without understanding the reason. For that, this PR creates a new editor that handles both XJSON and JSON format and does not changes the content that the user introduced while editing. Once the pipeline is saved, it will always retrieve the parsed data. I've created a whole new editor instead of modifying `XJsonEditor` because the original one is still in use in the `Custom processor`. This is because I've created [a list](https://github.com/SoniaSanzV/kibana/blob/d7d5ecafa7dbae96fe52c2e37394520b6353bd92/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts#L151-L157) of the processor fields that needs to be serialized in `convertProcessorInternalToProcessor`. Since we don't know the name of the custom processor, we can't apply this workflow. I'm not super happy with the idea of adding manually the names of the processors to a list. I would like to have something more agnostic but I haven't come up with a solution that allows me to do that without breaking other fields. So, any suggestions are super welcome! ### How to test Create one of the following processors: Foreach, Grok, Inference, Redact or Script. In the JSON field, add a JSON with both with and without escaped strings. While you are editing the processor before saving the pipeline, you should see the processor as you typed it. In the `Show Request` flyout, you should see the properly parsed JSON. The pipeline should save with the correct parsed values and, once saved, if you click update, you only will get parsed values. ### Demo https://github.com/user-attachments/assets/1f9681df-2fb4-4ed5-ac30-03f2937abfe9 --- .../__jest__/processors/foreach.test.tsx | 117 ++++++++++++++++++ .../__jest__/processors/grok.test.ts | 37 ++++++ .../__jest__/processors/inference.test.tsx | 111 +++++++++++++++++ .../__jest__/processors/processor.helpers.tsx | 8 +- .../__jest__/processors/redact.test.tsx | 36 ++++++ .../__jest__/processors/script.test.tsx | 104 ++++++++++++++++ .../field_components/xjson_editor.tsx | 13 +- .../processor_form.container.tsx | 2 +- .../processor_form/processors/custom.tsx | 20 +-- .../processor_form/processors/foreach.tsx | 11 +- .../processor_form/processors/grok.tsx | 14 ++- .../processor_form/processors/inference.tsx | 28 +++-- .../processor_form/processors/redact.tsx | 12 +- .../processor_form/processors/script.tsx | 19 ++- .../processor_form/processors/shared.test.ts | 46 ++++++- .../processor_form/processors/shared.ts | 58 ++++++++- .../components/pipeline_editor/serialize.ts | 4 +- .../components/pipeline_editor/utils.test.ts | 47 ++++++- .../components/pipeline_editor/utils.ts | 51 ++++++++ 19 files changed, 675 insertions(+), 63 deletions(-) create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx new file mode 100644 index 0000000000000..e4ca33fbffadb --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/foreach.test.tsx @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue, setupEnvironment } from './processor.helpers'; + +const FOREACH_TYPE = 'foreach'; + +describe('Processor: Foreach', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + const { httpSetup } = setupEnvironment(); + + beforeAll(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + // disable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = true; + }); + + afterAll(() => { + jest.useRealTimers(); + // enable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = false; + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup(httpSetup, { + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + + const { component, actions } = testBed; + + component.update(); + + // Open flyout to add new processor + actions.addProcessor(); + // Add type (the other fields are not visible until a type is selected) + await actions.addProcessorType(FOREACH_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is a required parameter + expect(form.getErrorsMessages()).toEqual(['A field value is required.']); + }); + + test('saves with default parameter values', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_foreach_processor'); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, FOREACH_TYPE); + + expect(processors[0][FOREACH_TYPE]).toEqual({ + field: 'test_foreach_processor', + }); + }); + + test('accepts processor definitions that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + form, + find, + component, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_foreach_processor'); + + await act(async () => { + find('processorField').simulate('change', { + jsonContent: '{"def_1":"""aaa"bbb""", "def_2":"aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, FOREACH_TYPE); + + expect(processors[0][FOREACH_TYPE]).toEqual({ + field: 'test_foreach_processor', + // eslint-disable-next-line prettier/prettier + processor: { def_1: 'aaa\"bbb', def_2: 'aaa(bbb' }, + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts index 3f98b95ba8b61..84e4948495695 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/grok.test.ts @@ -126,4 +126,41 @@ describe('Processor: Grok', () => { expect(processors[0][GROK_TYPE].patterns).toEqual([escapedValue]); }); + + test('accepts pattern definitions that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + form, + find, + component, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_grok_processor'); + + // Add pattern 1 + form.setInputValue('droppableList.input-0', 'pattern1'); + + await act(async () => { + find('patternDefinitionsField').simulate('change', { + jsonContent: '{"pattern_1":"""aaa"bbb""", "pattern_2":"aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, GROK_TYPE); + + expect(processors[0][GROK_TYPE]).toEqual({ + field: 'test_grok_processor', + patterns: ['pattern1'], + // eslint-disable-next-line prettier/prettier + pattern_definitions: { pattern_1: 'aaa\"bbb', pattern_2: 'aaa(bbb' }, + }); + }); }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx new file mode 100644 index 0000000000000..5a07fc63d087b --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/inference.test.tsx @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue, setupEnvironment } from './processor.helpers'; + +const INFERENCE_TYPE = 'inference'; + +describe('Processor: Script', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + const { httpSetup } = setupEnvironment(); + + beforeAll(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + // disable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = true; + }); + + afterAll(() => { + jest.useRealTimers(); + // enable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = false; + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup(httpSetup, { + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + + const { component, actions } = testBed; + + component.update(); + + // Open flyout to add new processor + actions.addProcessor(); + // Add type (the other fields are not visible until a type is selected) + await actions.addProcessorType(INFERENCE_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is a required parameter + expect(form.getErrorsMessages()).toEqual([ + 'A deployment, an inference, or a model ID value is required.', + ]); + }); + + test('accepts inference config and field maps that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + find, + form, + component, + } = testBed; + + form.setInputValue('inferenceModelId.input', 'test_inference_processor'); + + await act(async () => { + find('inferenceConfig').simulate('change', { + jsonContent: '{"inf_conf_1":"""aaa"bbb""", "inf_conf_2": "aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + await act(async () => { + find('fieldMap').simulate('change', { + jsonContent: '{"field_map_1":"""aaa"bbb""", "field_map_2": "aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, INFERENCE_TYPE); + + expect(processors[0][INFERENCE_TYPE]).toEqual({ + model_id: 'test_inference_processor', + // eslint-disable-next-line prettier/prettier + inference_config: { inf_conf_1: 'aaa\"bbb', inf_conf_2: 'aaa(bbb' }, + // eslint-disable-next-line prettier/prettier + field_map: { field_map_1: 'aaa\"bbb', field_map_2: 'aaa(bbb' }, + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx index 0f3ea833a3111..3eeff809999b7 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx @@ -199,4 +199,10 @@ type TestSubject = | 'ignoreMissingPipelineSwitch.input' | 'destinationField.input' | 'datasetField.input' - | 'namespaceField.input'; + | 'namespaceField.input' + | 'processorField' + | 'paramsField' + | 'scriptSource' + | 'inferenceModelId.input' + | 'inferenceConfig' + | 'fieldMap'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx index d34e4d1476bb8..eda4bac8266dd 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/redact.test.tsx @@ -138,4 +138,40 @@ describe('Processor: Redact', () => { pattern_definitions: { GITHUB_NAME: '@%{USERNAME}' }, }); }); + test('accepts pattern definitions that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + component, + find, + form, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_redact_processor'); + + // Add one pattern to the list + form.setInputValue('droppableList.input-0', 'pattern1'); + + await act(async () => { + find('patternDefinitionsField').simulate('change', { + jsonContent: '{"pattern_1":"""aaa"bbb""", "pattern_2":"aaa(bbb"}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, REDACT_TYPE); + + expect(processors[0][REDACT_TYPE]).toEqual({ + field: 'test_redact_processor', + patterns: ['pattern1'], + + pattern_definitions: { pattern_1: 'aaa"bbb', pattern_2: 'aaa(bbb' }, + }); + }); }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx new file mode 100644 index 0000000000000..428c87ea6a7e7 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/script.test.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue, setupEnvironment } from './processor.helpers'; + +const SCRIPT_TYPE = 'script'; + +describe('Processor: Script', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + const { httpSetup } = setupEnvironment(); + + beforeAll(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + // disable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = true; + }); + + afterAll(() => { + jest.useRealTimers(); + // enable all react-beautiful-dnd development warnings + (window as any)['__@hello-pangea/dnd-disable-dev-warnings'] = false; + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup(httpSetup, { + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + + const { component, actions } = testBed; + + component.update(); + + // Open flyout to add new processor + actions.addProcessor(); + // Add type (the other fields are not visible until a type is selected) + await actions.addProcessorType(SCRIPT_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is a required parameter + expect(form.getErrorsMessages()).toEqual(['A value is required.']); + }); + + test('accepts params that contains escaped characters', async () => { + const { + actions: { saveNewProcessor }, + find, + component, + } = testBed; + + await act(async () => { + find('scriptSource').simulate('change', { + jsonContent: 'ctx._source[params.sum_field]', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + await act(async () => { + find('paramsField').simulate('change', { + jsonContent: '{"sum_field":"""aaa"bbb"""}', + }); + + // advance timers to allow the form to validate + jest.advanceTimersByTime(0); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, SCRIPT_TYPE); + + expect(processors[0][SCRIPT_TYPE]).toEqual({ + source: 'ctx._source[params.sum_field]', + // eslint-disable-next-line prettier/prettier + params: { sum_field: 'aaa\"bbb' }, + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx index 5134df09ac93b..b5f98c260661d 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/xjson_editor.tsx @@ -7,9 +7,7 @@ import { XJsonLang } from '@kbn/monaco'; import React, { FunctionComponent, useCallback } from 'react'; -import { FieldHook, XJson } from '../../../../../../shared_imports'; - -const { useXJsonMode } = XJson; +import { FieldHook } from '../../../../../../shared_imports'; import { TextEditor } from './text_editor'; @@ -25,20 +23,17 @@ const defaultEditorOptions = { export const XJsonEditor: FunctionComponent = ({ field, editorProps }) => { const { value, setValue } = field; - const { xJson, setXJson, convertToJson } = useXJsonMode(value); - const onChange = useCallback( (s: any) => { - setXJson(s); - setValue(convertToJson(s)); + setValue(s); }, - [setValue, setXJson, convertToJson] + [setValue] ); return ( = ({ type: formState.type, fields: formState.customOptions ? { - ...formState.customOptions, + customOptions: formState.customOptions, } : { ...formState.fields, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx index 30cc54439f260..7aa55418552de 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/custom.tsx @@ -15,30 +15,20 @@ import { UseField, } from '../../../../../../shared_imports'; -const { emptyField, isJsonField } = fieldValidators; +const { emptyField } = fieldValidators; import { XJsonEditor } from '../field_components'; import { Fields } from '../processor_form.container'; -import { EDITOR_PX_HEIGHT } from './shared'; +import { EDITOR_PX_HEIGHT, from, isXJsonField, to } from './shared'; const customConfig: FieldConfig = { type: FIELD_TYPES.TEXT, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.customForm.optionsFieldLabel', { defaultMessage: 'Configuration', }), - serializer: (value: string) => { - try { - return JSON.parse(value); - } catch (error) { - // swallow error and return non-parsed value; - return value; - } - }, + serializer: from.optionalXJson, deserializer: (value: any) => { - if (value === '') { - return '{\n\n}'; - } - return JSON.stringify(value, null, 2); + return to.xJsonString(value.customOptions ? value.customOptions : value); }, validations: [ { @@ -52,7 +42,7 @@ const customConfig: FieldConfig = { ), }, { - validator: isJsonField( + validator: isXJsonField( i18n.translate('xpack.ingestPipelines.pipelineEditor.customForm.invalidJsonError', { defaultMessage: 'The input is not valid.', }) diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx index 55c3a3ac11155..2bcf2847ad77e 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/foreach.tsx @@ -13,16 +13,16 @@ import { FIELD_TYPES, fieldValidators, UseField } from '../../../../../../shared import { XJsonEditor } from '../field_components'; import { FieldNameField } from './common_fields/field_name_field'; -import { FieldsConfig, to, EDITOR_PX_HEIGHT } from './shared'; +import { FieldsConfig, to, EDITOR_PX_HEIGHT, from, isXJsonField } from './shared'; -const { emptyField, isJsonField } = fieldValidators; +const { emptyField } = fieldValidators; const fieldsConfig: FieldsConfig = { /* Required fields config */ processor: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: JSON.parse, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.foreachForm.processorFieldLabel', { defaultMessage: 'Processor', }), @@ -41,7 +41,7 @@ const fieldsConfig: FieldsConfig = { ), }, { - validator: isJsonField( + validator: isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.foreachForm.processorInvalidJsonError', { @@ -68,6 +68,7 @@ export const Foreach: FunctionComponent = () => { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'processorField', height: EDITOR_PX_HEIGHT.medium, 'aria-label': i18n.translate( 'xpack.ingestPipelines.pipelineEditor.foreachForm.optionsFieldAriaLabel', diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx index ae56d3b30a62d..f07fb32aa64bf 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx @@ -18,13 +18,13 @@ import { ArrayItem, } from '../../../../../../shared_imports'; -import { XJsonEditor, DragAndDropTextList } from '../field_components'; +import { DragAndDropTextList, XJsonEditor } from '../field_components'; import { FieldNameField } from './common_fields/field_name_field'; import { IgnoreMissingField } from './common_fields/ignore_missing_field'; -import { FieldsConfig, to, from, EDITOR_PX_HEIGHT } from './shared'; +import { FieldsConfig, to, from, EDITOR_PX_HEIGHT, isXJsonField } from './shared'; -const { isJsonField, emptyField } = fieldValidators; +const { emptyField } = fieldValidators; const i18nTexts = { addPatternLabel: i18n.translate( @@ -70,8 +70,8 @@ const fieldsConfig: FieldsConfig = { /* Optional field configs */ pattern_definitions: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: from.optionalJson, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.grokForm.patternDefinitionsLabel', { defaultMessage: 'Pattern definitions (optional)', }), @@ -84,7 +84,7 @@ const fieldsConfig: FieldsConfig = { ), validations: [ { - validator: isJsonField( + validator: isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.grokForm.patternsDefinitionsInvalidJSONError', { defaultMessage: 'Invalid JSON' } @@ -153,6 +153,7 @@ export const Grok: FunctionComponent = () => { config={fieldsConfig.pattern_definitions} componentProps={{ editorProps: { + 'data-test-subj': 'patternDefinitionsField', height: EDITOR_PX_HEIGHT.medium, 'aria-label': i18n.translate( 'xpack.ingestPipelines.pipelineEditor.grokForm.patternDefinitionsAriaLabel', @@ -163,6 +164,7 @@ export const Grok: FunctionComponent = () => { }, }} path="fields.pattern_definitions" + data-test-subj="patternDefinitions" /> { const documentationDocsLink = services.documentation.getDocumentationUrl(); return ( <> - + { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'fieldMap', height: EDITOR_PX_HEIGHT.medium, options: { minimap: { enabled: false } }, }, @@ -173,6 +178,7 @@ export const Inference: FunctionComponent = () => { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'inferenceConfig', height: EDITOR_PX_HEIGHT.medium, options: { minimap: { enabled: false } }, }, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx index 743a5b6b7275a..37e488996dbe8 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/redact.tsx @@ -19,13 +19,13 @@ import { ValidationFunc, } from '../../../../../../shared_imports'; -import { XJsonEditor, InputList } from '../field_components'; +import { InputList, XJsonEditor } from '../field_components'; import { FieldNameField } from './common_fields/field_name_field'; import { IgnoreMissingField } from './common_fields/ignore_missing_field'; -import { FieldsConfig, to, from, EDITOR_PX_HEIGHT } from './shared'; +import { FieldsConfig, to, from, EDITOR_PX_HEIGHT, isXJsonField } from './shared'; -const { isJsonField, emptyField } = fieldValidators; +const { emptyField } = fieldValidators; const i18nTexts = { addPatternLabel: i18n.translate( @@ -68,8 +68,8 @@ const fieldsConfig: FieldsConfig = { /* Optional field configs */ pattern_definitions: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: from.optionalJson, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.redactForm.patternDefinitionsLabel', { @@ -85,7 +85,7 @@ const fieldsConfig: FieldsConfig = { ), validations: [ { - validator: isJsonField( + validator: isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.redactForm.patternsDefinitionsInvalidJSONError', { defaultMessage: 'Invalid JSON' } diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx index 18da4097eaf2c..bdbf909716219 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/script.tsx @@ -21,9 +21,16 @@ import { import { XJsonEditor, TextEditor } from '../field_components'; -import { FieldsConfig, to, from, FormFieldsComponent, EDITOR_PX_HEIGHT } from './shared'; +import { + FieldsConfig, + to, + from, + FormFieldsComponent, + EDITOR_PX_HEIGHT, + isXJsonField, +} from './shared'; -const { isJsonField, emptyField } = fieldValidators; +const { emptyField } = fieldValidators; const fieldsConfig: FieldsConfig = { /* Required fields config */ @@ -98,8 +105,8 @@ const fieldsConfig: FieldsConfig = { params: { type: FIELD_TYPES.TEXT, - deserializer: to.jsonString, - serializer: from.optionalJson, + deserializer: to.xJsonString, + serializer: from.optionalXJson, label: i18n.translate('xpack.ingestPipelines.pipelineEditor.scriptForm.paramsFieldLabel', { defaultMessage: 'Parameters', }), @@ -113,7 +120,7 @@ const fieldsConfig: FieldsConfig = { { validator: (value) => { if (value.value) { - return isJsonField( + return isXJsonField( i18n.translate( 'xpack.ingestPipelines.pipelineEditor.scriptForm.processorInvalidJsonError', { @@ -166,6 +173,7 @@ export const Script: FormFieldsComponent = ({ initialFieldValues }) => { component={TextEditor} componentProps={{ editorProps: { + 'data-test-subj': 'scriptSource', languageId: scriptLanguage, suggestionProvider: scriptLanguage === PainlessLang.ID ? suggestionProvider : undefined, @@ -191,6 +199,7 @@ export const Script: FormFieldsComponent = ({ initialFieldValues }) => { component={XJsonEditor} componentProps={{ editorProps: { + 'data-test-subj': 'paramsField', height: EDITOR_PX_HEIGHT.medium, 'aria-label': i18n.translate( 'xpack.ingestPipelines.pipelineEditor.scriptForm.paramsFieldAriaLabel', diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts index 4b01f22a9383d..a3b77293c66fd 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { from, to } from './shared'; +import { ValidationFuncArg } from '@kbn/console-plugin/public/shared_imports'; +import { from, isXJsonField, to } from './shared'; describe('shared', () => { describe('deserialization helpers', () => { @@ -28,6 +29,21 @@ describe('shared', () => { '%{clientip} %{ident} %{auth} [%{@timestamp}] \\"%{verb} %{request} HTTP/%{httpversion}\\" %{status} %{size}' ); }); + test('to.xJsonString', () => { + const input1 = ''; + expect(to.xJsonString(input1)).toBe('{}'); + + // eslint-disable-next-line prettier/prettier + const input2 = '{"ISSUE": "aaa\"bbb","ISSUE2": "aaa\\(bbb","ISSUE3": """aaa\"bbb"""}'; + expect(to.xJsonString(input2)).toBe( + // eslint-disable-next-line prettier/prettier + '{"ISSUE": "aaa\"bbb","ISSUE2": "aaa\\(bbb","ISSUE3": """aaa\"bbb"""}' + ); + + // eslint-disable-next-line prettier/prettier + const input3 = { ISSUE: "aaa\"bbb", ISSUE2: "aaa\\(bbb" }; + expect(to.xJsonString(input3)).toBe(JSON.stringify(input3, null, 2)); + }); }); describe('serialization helpers', () => { @@ -49,5 +65,33 @@ describe('shared', () => { `%{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size}` ); }); + test('from.optionalXJson', () => { + const input1 = ''; + expect(from.optionalXJson(input1)).toBe(undefined); + + const input2 = '{}'; + expect(from.optionalXJson(input2)).toBe(undefined); + + const input3 = '{"ISSUE": "aaa","ISSUE2": "bbb"}'; + expect(from.optionalXJson(input3)).toBe(input3); + }); + }); + describe('validators', () => { + test('isXJsonField', () => { + const message = 'test error message'; + const code = 'ERR_JSON_FORMAT'; + + const validate = isXJsonField(message); + const validator = (value: unknown) => validate({ value } as ValidationFuncArg); + + // Valid JSON + const input1 = '{"ISSUE": """aaa"bbb""", "ISSUE2": """aaa\bbb"""}'; + expect(validator(input1)).toBeUndefined(); + + // Invalid JSON + // eslint-disable-next-line prettier/prettier + const input2 = '{"ISSUE": """"aaa\"bbb""'; + expect(validator(input2)).toMatchObject({ message, code }); + }); }); }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts index a14944a33a8ce..f6f7f2b105de9 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts @@ -10,9 +10,11 @@ import * as rt from 'io-ts'; import { i18n } from '@kbn/i18n'; import { isRight } from 'fp-ts/lib/Either'; +import { ERROR_CODE } from '@kbn/es-ui-shared-plugin/static/forms/helpers/field_validators/types'; import { FieldConfig, ValidationFunc, fieldValidators } from '../../../../../../shared_imports'; +import { collapseEscapedStrings } from '../../../utils'; -const { emptyField } = fieldValidators; +const { emptyField, isJsonField } = fieldValidators; export const arrayOfStrings = rt.array(rt.string); @@ -21,6 +23,35 @@ export function isArrayOfStrings(v: unknown): v is string[] { return isRight(res); } +/** + * Format a XJson string input as parsed JSON. Replaces the invalid characters + * with a placeholder, parses the new string in a JSON format with the expected + * indentantion and then replaces the placeholders with the original values. + */ +const formatXJsonString = (input: string) => { + let placeholder = 'PLACEHOLDER'; + const INVALID_STRING_REGEX = /"""(.*?)"""/gs; + while (input.includes(placeholder)) { + placeholder += '_'; + } + const modifiedInput = input.replace(INVALID_STRING_REGEX, () => `"${placeholder}"`); + + let jsonObject; + try { + jsonObject = JSON.parse(modifiedInput); + } catch (error) { + return input; + } + let formattedJsonString = JSON.stringify(jsonObject, null, 2); + const invalidStrings = input.match(INVALID_STRING_REGEX); + if (invalidStrings) { + invalidStrings.forEach((invalidString) => { + formattedJsonString = formattedJsonString.replace(`"${placeholder}"`, invalidString); + }); + } + return formattedJsonString; +}; + /** * Shared deserializer functions. * @@ -50,6 +81,15 @@ export const to = { } return v; }, + xJsonString: (v: unknown) => { + if (!v) { + return '{}'; + } + if (typeof v === 'string') { + return formatXJsonString(v); + } + return JSON.stringify(v, null, 2); + }, }; /** @@ -98,6 +138,12 @@ export const from = { } } }, + optionalXJson: (v: string) => { + if (v && v !== '{}') { + return v; + } + return undefined; + }, }; const isJSONString = (v: string) => { @@ -120,6 +166,16 @@ export const isJSONStringValidator: ValidationFunc = ({ value }) => { } }; +export const isXJsonField = + (message: string, { allowEmptyString = false }: { allowEmptyString?: boolean } = {}) => + (...args: Parameters): ReturnType> => { + const [{ value, ...rest }] = args; + return isJsonField(message, { allowEmptyString })({ + ...rest, + value: collapseEscapedStrings(value as string), + }); + }; + /** * Similar to the emptyField validator but we accept whitespace characters. */ diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts index e0a8fb49d5d01..3edc4ad02d6a0 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/serialize.ts @@ -8,6 +8,7 @@ import { Processor } from '../../../../common/types'; import { ProcessorInternal } from './types'; +import { convertProccesorsToJson } from './utils'; interface SerializeArgs { /** @@ -33,9 +34,10 @@ const convertProcessorInternalToProcessor = ( copyIdToTag?: boolean ): Processor => { const { options, onFailure, type, id } = processor; + const outProcessor = { [type]: { - ...options, + ...convertProccesorsToJson(options), }, }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts index 6e367a83bf8d4..8b75afd985785 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.test.ts @@ -5,7 +5,13 @@ * 2.0. */ -import { getValue, setValue, hasTemplateSnippet } from './utils'; +import { + getValue, + setValue, + hasTemplateSnippet, + collapseEscapedStrings, + convertProccesorsToJson, +} from './utils'; describe('get and set values', () => { const testObject = Object.freeze([{ onFailure: [{ onFailure: 1 }] }]); @@ -51,3 +57,42 @@ describe('template snippets', () => { expect(hasTemplateSnippet('{{{hello.world}}}')).toBe(true); }); }); + +describe('collapse escaped strings', () => { + it('returns escaped literal strings', () => { + expect(collapseEscapedStrings('{"1": """aaa\bbb""", "2": """ccc"""}')).toBe( + '{"1": "aaa\\bbb", "2": "ccc"}' + ); + }); +}); + +describe('convert processors to json', () => { + it('returns converted processors', () => { + const obj = { + field1: 'mustNotChange', + field2: 123, + field3: '{1: "mustNotChange"}', + pattern_definitions: '{"1": """aaa"bbb"""}', + processor: '{"1": """aaa"bbb"""}', + inference_config: '{"1": """aaa"bbb"""}', + field_map: '{"1": """aaa"bbb"""}', + customOptions: '{"customProcessor": """aaa"bbb"""}', + }; + + expect(convertProccesorsToJson(obj)).toEqual({ + field1: 'mustNotChange', + field2: 123, + field3: '{1: "mustNotChange"}', + // eslint-disable-next-line prettier/prettier + pattern_definitions: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + processor: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + inference_config: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + field_map: { 1: "aaa\"bbb" }, + // eslint-disable-next-line prettier/prettier + customProcessor: "aaa\"bbb" + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts index d7e6fc4a4d9ac..533f9621bbec9 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/utils.ts @@ -119,3 +119,54 @@ export const hasTemplateSnippet = (str: string = '') => { // * And followed by }}} return /{{{.+}}}/.test(str); }; + +const escapeLiteralStrings = (data: string): string[] => { + const splitData = data.split(`"""`); + for (let i = 1; i < splitData.length - 1; i += 2) { + splitData[i] = JSON.stringify(splitData[i]); + } + return splitData; +}; + +const convertProcessorValueToJson = (data: string): any => { + if (!data) { + return undefined; + } + + try { + const escapedData = escapeLiteralStrings(data); + return JSON.parse(escapedData.join('')); + } catch (error) { + return data; + } +}; + +export const collapseEscapedStrings = (data: string): string => { + if (data) { + return escapeLiteralStrings(data).join(''); + } + return data; +}; + +const fieldToConvertToJson = [ + 'inference_config', + 'field_map', + 'params', + 'pattern_definitions', + 'processor', +]; + +export const convertProccesorsToJson = (obj: { [key: string]: any }): { [key: string]: any } => { + return Object.fromEntries( + Object.entries(obj).flatMap(([key, value]) => { + if (key === 'customOptions') { + const convertedValue = convertProcessorValueToJson(value); + return Object.entries(convertedValue); + } else if (fieldToConvertToJson.includes(key)) { + return [[key, convertProcessorValueToJson(value)]]; + } else { + return [[key, value]]; + } + }) + ); +};