[Ingest Pipelines] Fix serialization and deserialization of user inpu…

…t for "patterns" fields (elastic#94689) (elastic#94897) * updated serialization and deserialization behavior of dissect and gsub processors, also addded a test * also fix grok processor * pivot input checking to use JSON.stringify and JSON.parse Co-authored-by: Kibana Machine <[email protected]> Co-authored-by: Kibana Machine <[email protected]>
dokmic · Mar 18, 2021 · 1e7935e · 1e7935e
1 parent 68e8ba4
commit 1e7935e
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 12 deletions.
diff --git a/...ts/pipeline_editor/components/processor_form/field_components/drag_and_drop_text_list.tsx b/...ts/pipeline_editor/components/processor_form/field_components/drag_and_drop_text_list.tsx
@@ -44,7 +44,15 @@ interface Props {
   /**
    * Validation to be applied to every text item
    */
-  textValidation?: ValidationFunc<any, string, string>;
+  textValidations?: Array<ValidationFunc<any, string, string>>;
+  /**
+   * Serializer to be applied to every text item
+   */
+  textSerializer?: <O = string>(v: string) => O;
+  /**
+   * Deserializer to be applied to every text item
+   */
+  textDeserializer?: (v: unknown) => string;
 }
 
 const i18nTexts = {
@@ -63,7 +71,9 @@ function DragAndDropTextListComponent({
   onAdd,
   onRemove,
   addLabel,
-  textValidation,
+  textValidations,
+  textDeserializer,
+  textSerializer,
 }: Props): JSX.Element {
   const [droppableId] = useState(() => uuid.v4());
   const [firstItemId] = useState(() => uuid.v4());
@@ -133,9 +143,11 @@ function DragAndDropTextListComponent({
                             <UseField<string>
                               path={item.path}
                               config={{
-                                validations: textValidation
-                                  ? [{ validator: textValidation }]
+                                validations: textValidations
+                                  ? textValidations.map((validator) => ({ validator }))
                                   : undefined,
+                                deserializer: textDeserializer,
+                                serializer: textSerializer,
                               }}
                               readDefaultValueOnForm={!item.isNew}
                             >

diff --git a/...c/application/components/pipeline_editor/components/processor_form/processors/dissect.tsx b/...c/application/components/pipeline_editor/components/processor_form/processors/dissect.tsx
@@ -22,7 +22,7 @@ import {
 
 import { FieldNameField } from './common_fields/field_name_field';
 import { IgnoreMissingField } from './common_fields/ignore_missing_field';
-import { EDITOR_PX_HEIGHT, from } from './shared';
+import { EDITOR_PX_HEIGHT, from, to, isJSONStringValidator } from './shared';
 
 const { emptyField } = fieldValidators;
 
@@ -34,6 +34,8 @@ const getFieldsConfig = (esDocUrl: string): Record<string, FieldConfig> => {
       label: i18n.translate('xpack.ingestPipelines.pipelineEditor.dissectForm.patternFieldLabel', {
         defaultMessage: 'Pattern',
       }),
+      deserializer: to.escapeBackslashes,
+      serializer: from.unescapeBackslashes,
       helpText: (
         <FormattedMessage
           id="xpack.ingestPipelines.pipelineEditor.dissectForm.patternFieldHelpText"
@@ -67,6 +69,9 @@ const getFieldsConfig = (esDocUrl: string): Record<string, FieldConfig> => {
             )
           ),
         },
+        {
+          validator: isJSONStringValidator,
+        },
       ],
     },
     /* Optional field config */

diff --git a/...blic/application/components/pipeline_editor/components/processor_form/processors/grok.tsx b/...blic/application/components/pipeline_editor/components/processor_form/processors/grok.tsx
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { flow } from 'lodash';
 import React, { FunctionComponent } from 'react';
 import { i18n } from '@kbn/i18n';
 
@@ -22,7 +23,7 @@ import { XJsonEditor, DragAndDropTextList } from '../field_components';
 
 import { FieldNameField } from './common_fields/field_name_field';
 import { IgnoreMissingField } from './common_fields/ignore_missing_field';
-import { FieldsConfig, to, from, EDITOR_PX_HEIGHT } from './shared';
+import { FieldsConfig, to, from, EDITOR_PX_HEIGHT, isJSONStringValidator } from './shared';
 
 const { isJsonField, emptyField } = fieldValidators;
 
@@ -46,14 +47,19 @@ const patternsValidation: ValidationFunc<any, string, ArrayItem[]> = ({ value, f
   }
 };
 
-const patternValidation = emptyField(valueRequiredMessage);
+const patternValidations: Array<ValidationFunc<any, string, string>> = [
+  emptyField(valueRequiredMessage),
+  isJSONStringValidator,
+];
 
 const fieldsConfig: FieldsConfig = {
   /* Required field configs */
   patterns: {
     label: i18n.translate('xpack.ingestPipelines.pipelineEditor.grokForm.patternsFieldLabel', {
       defaultMessage: 'Patterns',
     }),
+    deserializer: flow(String, to.escapeBackslashes),
+    serializer: from.unescapeBackslashes,
     helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.grokForm.patternsHelpText', {
       defaultMessage:
         'Grok expressions used to match and extract named capture groups. Uses the first matching expression.',
@@ -133,7 +139,9 @@ export const Grok: FunctionComponent = () => {
               onAdd={addItem}
               onRemove={removeItem}
               addLabel={i18nTexts.addPatternLabel}
-              textValidation={patternValidation}
+              textValidations={patternValidations}
+              textDeserializer={fieldsConfig.patterns?.deserializer}
+              textSerializer={fieldsConfig.patterns?.serializer}
             />
           );
         }}

diff --git a/...blic/application/components/pipeline_editor/components/processor_form/processors/gsub.tsx b/...blic/application/components/pipeline_editor/components/processor_form/processors/gsub.tsx
@@ -5,14 +5,15 @@
  * 2.0.
  */
 
+import { flow } from 'lodash';
 import React, { FunctionComponent } from 'react';
 import { i18n } from '@kbn/i18n';
 
 import { FIELD_TYPES, fieldValidators, UseField, Field } from '../../../../../../shared_imports';
 
 import { TextEditor } from '../field_components';
 
-import { EDITOR_PX_HEIGHT, FieldsConfig } from './shared';
+import { EDITOR_PX_HEIGHT, FieldsConfig, from, to, isJSONStringValidator } from './shared';
 import { FieldNameField } from './common_fields/field_name_field';
 import { IgnoreMissingField } from './common_fields/ignore_missing_field';
 import { TargetField } from './common_fields/target_field';
@@ -26,7 +27,8 @@ const fieldsConfig: FieldsConfig = {
     label: i18n.translate('xpack.ingestPipelines.pipelineEditor.gsubForm.patternFieldLabel', {
       defaultMessage: 'Pattern',
     }),
-    deserializer: String,
+    deserializer: flow(String, to.escapeBackslashes),
+    serializer: from.unescapeBackslashes,
     helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.gsubForm.patternFieldHelpText', {
       defaultMessage: 'Regular expression used to match substrings in the field.',
     }),
@@ -38,6 +40,9 @@ const fieldsConfig: FieldsConfig = {
           })
         ),
       },
+      {
+        validator: isJSONStringValidator,
+      },
     ],
   },
 

diff --git a/...pplication/components/pipeline_editor/components/processor_form/processors/shared.test.ts b/...pplication/components/pipeline_editor/components/processor_form/processors/shared.test.ts
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { from, to } from './shared';
+
+describe('shared', () => {
+  describe('deserialization helpers', () => {
+    // This is the text that will be passed to the text input
+    test('to.escapeBackslashes', () => {
+      // this input loaded from the server
+      const input1 = 'my\ttab';
+      expect(to.escapeBackslashes(input1)).toBe('my\\ttab');
+
+      // this input loaded from the server
+      const input2 = 'my\\ttab';
+      expect(to.escapeBackslashes(input2)).toBe('my\\\\ttab');
+
+      // this input loaded from the server
+      const input3 = '\t\n\rOK';
+      expect(to.escapeBackslashes(input3)).toBe('\\t\\n\\rOK');
+
+      const input4 = `%{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size}`;
+      expect(to.escapeBackslashes(input4)).toBe(
+        '%{clientip} %{ident} %{auth} [%{@timestamp}] \\"%{verb} %{request} HTTP/%{httpversion}\\" %{status} %{size}'
+      );
+    });
+  });
+
+  describe('serialization helpers', () => {
+    test('from.unescapeBackslashes', () => {
+      // user typed in "my\ttab"
+      const input1 = 'my\\ttab';
+      expect(from.unescapeBackslashes(input1)).toBe('my\ttab');
+
+      // user typed in "my\\tab"
+      const input2 = 'my\\\\ttab';
+      expect(from.unescapeBackslashes(input2)).toBe('my\\ttab');
+
+      // user typed in "\t\n\rOK"
+      const input3 = '\\t\\n\\rOK';
+      expect(from.unescapeBackslashes(input3)).toBe('\t\n\rOK');
+
+      const input5 = `%{clientip} %{ident} %{auth} [%{@timestamp}] \\"%{verb} %{request} HTTP/%{httpversion}\\" %{status} %{size}`;
+      expect(from.unescapeBackslashes(input5)).toBe(
+        `%{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size}`
+      );
+    });
+  });
+});
diff --git a/...lic/application/components/pipeline_editor/components/processor_form/processors/shared.ts b/...lic/application/components/pipeline_editor/components/processor_form/processors/shared.ts
@@ -5,11 +5,12 @@
  * 2.0.
  */
 
-import { FunctionComponent } from 'react';
+import type { FunctionComponent } from 'react';
 import * as rt from 'io-ts';
+import { i18n } from '@kbn/i18n';
 import { isRight } from 'fp-ts/lib/Either';
 
-import { FieldConfig } from '../../../../../../shared_imports';
+import { FieldConfig, ValidationFunc } from '../../../../../../shared_imports';
 
 export const arrayOfStrings = rt.array(rt.string);
 
@@ -36,6 +37,17 @@ export const to = {
   arrayOfStrings: (v: unknown): string[] =>
     isArrayOfStrings(v) ? v : typeof v === 'string' && v.length ? [v] : [],
   jsonString: (v: unknown) => (v ? JSON.stringify(v, null, 2) : '{}'),
+  /**
+   * Useful when deserializing strings that will be rendered inside of text areas or text inputs. We want
+   * a string like: "my\ttab" to render the same, not to render as "my<tab>tab".
+   */
+  escapeBackslashes: (v: unknown) => {
+    if (typeof v === 'string') {
+      const s = JSON.stringify(v);
+      return s.slice(1, s.length - 1);
+    }
+    return v;
+  },
 };
 
 /**
@@ -69,6 +81,41 @@ export const from = {
   optionalArrayOfStrings: (v: string[]) => (v.length ? v : undefined),
   undefinedIfValue: (value: unknown) => (v: boolean) => (v === value ? undefined : v),
   emptyStringToUndefined: (v: unknown) => (v === '' ? undefined : v),
+  /**
+   * Useful when serializing user input from a <textarea /> that we want to later JSON.stringify but keep the same as what
+   * the user input. For instance, given "my\ttab", encoded as "my%5Ctab" will JSON.stringify to "my\\ttab", instead we want
+   * to keep the input exactly as the user entered it.
+   */
+  unescapeBackslashes: (v: unknown) => {
+    if (typeof v === 'string') {
+      try {
+        return JSON.parse(`"${v}"`);
+      } catch (e) {
+        // Best effort
+        return v;
+      }
+    }
+  },
+};
+
+const isJSONString = (v: string) => {
+  try {
+    JSON.parse(`"${v}"`);
+    return true;
+  } catch (e) {
+    return false;
+  }
+};
+
+export const isJSONStringValidator: ValidationFunc = ({ value }) => {
+  if (typeof value !== 'string' || !isJSONString(value)) {
+    return {
+      message: i18n.translate(
+        'xpack.ingestPipelines.pipelineEditor.jsonStringField.invalidStringMessage',
+        { defaultMessage: 'Invalid JSON string.' }
+      ),
+    };
+  }
 };
 
 export const EDITOR_PX_HEIGHT = {