From 0eefaf8cbc261a2e044fe830ac2d955513b37e47 Mon Sep 17 00:00:00 2001
From: David Roberts <dave.roberts@elastic.co>
Date: Tue, 22 Feb 2022 17:56:02 +0000
Subject: [PATCH] [ML] Text structure finder caps exclude lines pattern at 1000
 characters (#84236)

Because of the way Filebeat parses CSV files the text structure finder
needs to generate a regular expression that will ignore the header row
of the CSV file.

It does this by concatenating the column names separated by the delimiter
with optional quoting. However, if there are hundreds of columns this can
lead to a very long regular expression, potentially one that cannot be
evaluated by some programming languages.

This change limits the length of the regular expression to 1000 characters
by only including elements for the first few columns when there are many.
Matching 1000 characters of header should be sufficient to reliably
identify the header row even when it is much longer. It is extremely
unlikely that there would be a data row where the first 1000 characters
exactly matched the header but then subsequent fields diverged.

Fixes #83434
---
 docs/changelog/84236.yaml                     |  6 +++
 .../DelimitedTextStructureFinder.java         | 49 ++++++++++++++-----
 .../DelimitedTextStructureFinderTests.java    | 23 +++++++++
 3 files changed, 66 insertions(+), 12 deletions(-)
 create mode 100644 docs/changelog/84236.yaml

diff --git a/docs/changelog/84236.yaml b/docs/changelog/84236.yaml
new file mode 100644
index 0000000000000..319dbc312b5bf
--- /dev/null
+++ b/docs/changelog/84236.yaml
@@ -0,0 +1,6 @@
+pr: 84236
+summary: Text structure finder caps exclude lines pattern at 1000 characters
+area: Machine Learning
+type: bug
+issues:
+ - 83434
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinder.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinder.java
index 6d54fe73c95cd..e1c425c819639 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinder.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinder.java
@@ -33,7 +33,8 @@
 
 public class DelimitedTextStructureFinder implements TextStructureFinder {
 
-    private static final String REGEX_NEEDS_ESCAPE_PATTERN = "([\\\\|()\\[\\]{}^$.+*?])";
+    static final int MAX_EXCLUDE_LINES_PATTERN_LENGTH = 1000;
+    static final String REGEX_NEEDS_ESCAPE_PATTERN = "([\\\\|()\\[\\]{}^$.+*?])";
     private static final int MAX_LEVENSHTEIN_COMPARISONS = 100;
     private static final int LONG_FIELD_THRESHOLD = 100;
     private final List<String> sampleMessages;
@@ -137,20 +138,11 @@ static DelimitedTextStructureFinder makeDelimitedTextStructureFinder(
             .setColumnNames(columnNamesList);
 
         String quote = String.valueOf(quoteChar);
-        String twoQuotes = quote + quote;
         String quotePattern = quote.replaceAll(REGEX_NEEDS_ESCAPE_PATTERN, "\\\\$1");
         String optQuotePattern = quotePattern + "?";
         String delimiterPattern = (delimiter == '\t') ? "\\t" : String.valueOf(delimiter).replaceAll(REGEX_NEEDS_ESCAPE_PATTERN, "\\\\$1");
         if (isHeaderInText) {
-            structureBuilder.setExcludeLinesPattern(
-                "^"
-                    + Arrays.stream(header)
-                        .map(
-                            column -> optQuotePattern + column.replace(quote, twoQuotes).replaceAll(REGEX_NEEDS_ESCAPE_PATTERN, "\\\\$1")
-                                + optQuotePattern
-                        )
-                        .collect(Collectors.joining(delimiterPattern))
-            );
+            structureBuilder.setExcludeLinesPattern(makeExcludeLinesPattern(header, quote, optQuotePattern, delimiterPattern));
         }
 
         if (trimFields) {
@@ -413,7 +405,7 @@ private static boolean isFirstRowUnusual(List<String> explanation, List<List<Str
         for (int i = 0; numComparisons < MAX_LEVENSHTEIN_COMPARISONS && i < otherRowStrs.size(); ++i) {
             for (int j = i + 1 + random.nextInt(innerIncrement); numComparisons < MAX_LEVENSHTEIN_COMPARISONS
                 && j < otherRowStrs.size(); j += innerIncrement) {
-                otherRowStats.accept((double) levenshteinFieldwiseCompareRows(otherRows.get(i), otherRows.get(j), shortFieldMask));
+                otherRowStats.accept(levenshteinFieldwiseCompareRows(otherRows.get(i), otherRows.get(j), shortFieldMask));
                 ++numComparisons;
             }
         }
@@ -813,4 +805,37 @@ static String makeMultilineStartPattern(
         explanation.add("Failed to create a suitable multi-line start pattern");
         return null;
     }
+
+    /**
+     * Make a regular expression that Filebeat can use to ignore the header line of the delimited file.
+     * (Such lines may be observed multiple times if multiple delimited files are concatenated.)
+     *
+     * This pattern consists of a pattern that matches the literal column names, optionally quoted and
+     * separated by the delimiter.
+     *
+     * In the event that the column names are long and/or numerous only the first few are included.
+     * These ought to be enough to reliably distinguish the header line from data lines.
+     */
+    static String makeExcludeLinesPattern(String[] header, String quote, String optQuotePattern, String delimiterPattern) {
+        String twoQuotes = quote + quote;
+        StringBuilder excludeLinesPattern = new StringBuilder("^");
+        boolean isFirst = true;
+        int maxLengthOfFields = MAX_EXCLUDE_LINES_PATTERN_LENGTH - delimiterPattern.length() - 2; // 2 is length of ".*"
+        for (String column : header) {
+            String columnPattern = optQuotePattern + column.replace(quote, twoQuotes).replaceAll(REGEX_NEEDS_ESCAPE_PATTERN, "\\\\$1")
+                + optQuotePattern;
+            if (isFirst) {
+                // Always append the pattern for the first column, even if it exceeds the limit
+                excludeLinesPattern.append(columnPattern);
+                isFirst = false;
+            } else {
+                if (excludeLinesPattern.length() + columnPattern.length() > maxLengthOfFields) {
+                    excludeLinesPattern.append(".*");
+                    break;
+                }
+                excludeLinesPattern.append(delimiterPattern).append(columnPattern);
+            }
+        }
+        return excludeLinesPattern.toString();
+    }
 }
diff --git a/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinderTests.java b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinderTests.java
index 3530c94c8ede7..9b94ed02515a6 100644
--- a/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinderTests.java
+++ b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/DelimitedTextStructureFinderTests.java
@@ -25,9 +25,12 @@
 import static org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinder.stringToNumberPosBitSet;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
 
 public class DelimitedTextStructureFinderTests extends TextStructureTestCase {
 
@@ -1122,6 +1125,26 @@ public void testMultilineStartPatternDeterminationTooHard() {
         assertThat(explanation, contains("Failed to create a suitable multi-line start pattern"));
     }
 
+    public void testMakeExcludeLinesPattern() {
+
+        String[] header = generateRandomStringArray(1000, randomIntBetween(5, 50), false, false);
+        String quote = randomFrom("\"", "'");
+        String quotePattern = quote.replaceAll(DelimitedTextStructureFinder.REGEX_NEEDS_ESCAPE_PATTERN, "\\\\$1");
+        String optQuotePattern = quotePattern + "?";
+        char delimiter = randomFrom(',', ';', '\t', '|');
+        String delimiterPattern = (delimiter == '\t')
+            ? "\\t"
+            : String.valueOf(delimiter).replaceAll(DelimitedTextStructureFinder.REGEX_NEEDS_ESCAPE_PATTERN, "\\\\$1");
+
+        String excludeLinesPattern = DelimitedTextStructureFinder.makeExcludeLinesPattern(header, quote, optQuotePattern, delimiterPattern);
+
+        assertThat(excludeLinesPattern, startsWith("^"));
+        assertThat(excludeLinesPattern.length(), lessThanOrEqualTo(DelimitedTextStructureFinder.MAX_EXCLUDE_LINES_PATTERN_LENGTH));
+        if (excludeLinesPattern.contains(header[header.length - 1]) == false) {
+            assertThat(excludeLinesPattern, endsWith(".*"));
+        }
+    }
+
     static Map<String, Object> randomCsvProcessorSettings() {
         String field = randomAlphaOfLength(10);
         return DelimitedTextStructureFinder.makeCsvProcessorSettings(