Strip quotes of eval field names, add test for quoted fields in evalT…

…est, add test for rename with single and double quotes (#430)
teragrep · Dec 3, 2024 · fa56c50 · fa56c50
1 parent eb9114a
commit fa56c50
Show file tree

Hide file tree

Showing 3 changed files with 70 additions and 1 deletion.
diff --git a/src/main/java/com/teragrep/pth10/steps/eval/EvalStep.java b/src/main/java/com/teragrep/pth10/steps/eval/EvalStep.java
@@ -45,6 +45,8 @@
  */
 package com.teragrep.pth10.steps.eval;
 
+import com.teragrep.pth10.ast.TextString;
+import com.teragrep.pth10.ast.UnquotedText;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.functions;
@@ -78,7 +80,7 @@ public Dataset<Row> get(Dataset<Row> dataset) {
         }
 
         // perform eval and long->ts
-        ds = ds.withColumn(leftSide, rightSide);
+        ds = ds.withColumn(new UnquotedText(new TextString(leftSide)).read(), rightSide);
         if (timeColumnExists) {
             ds = ds.withColumn("_time", functions.from_unixtime(functions.col("_time")).cast(DataTypes.TimestampType));
         }

diff --git a/src/test/java/com/teragrep/pth10/RenameTransformationTest.java b/src/test/java/com/teragrep/pth10/RenameTransformationTest.java
@@ -119,4 +119,35 @@ public void testRenameMultipleFields() {
                         }
                 );
     }
+
+    @Test
+    @DisabledIfSystemProperty(
+            named = "skipSparkTest",
+            matches = "true"
+    )
+    public void rename_DoubleQuotes_test() {
+        streamingTestUtil
+                .performDPLTest(
+                        "| makeresults count=1 | eval \"a\" = \"something\" | rename \"a\" as \"b\"", testFile, ds -> {
+                            Assertions
+                                    .assertEquals("[_time, b]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !");
+                        }
+                );
+    }
+
+    @Test
+    @DisabledIfSystemProperty(
+            named = "skipSparkTest",
+            matches = "true"
+    )
+    public void rename_SingleQuotes_test() {
+        streamingTestUtil
+                .performDPLTest(
+                        "| makeresults count=1 | eval 'abc(def)' = \"xyz\" | rename 'abc(def)' as '(foo)bar'", testFile,
+                        ds -> {
+                            Assertions
+                                    .assertEquals("[_time, (foo)bar]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !");
+                        }
+                );
+    }
 }
diff --git a/src/test/java/com/teragrep/pth10/evalTest.java b/src/test/java/com/teragrep/pth10/evalTest.java
@@ -3589,4 +3589,40 @@ public void evalAfterSpath_ComparisonTest() {
             Assertions.assertEquals(expected, a);
         });
     }
+
+    @Test
+    @DisabledIfSystemProperty(
+            named = "skipSparkTest",
+            matches = "true"
+    )
+    public void evalFieldWithDoubleQuotes() {
+        String query = "index=index_A | eval \"quotedField\" = \"string\"";
+        String schema = "StructType(StructField(_raw,StringType,true),StructField(_time,TimestampType,true),StructField(host,StringType,true),StructField(id,LongType,true),StructField(index,StringType,true),StructField(offset,LongType,true),StructField(partition,StringType,true),StructField(source,StringType,true),StructField(sourcetype,StringType,true),StructField(quotedField,StringType,false))";
+        String testFile = "src/test/resources/spath/spathTransformationTest_numeric2*.jsonl";
+
+        streamingTestUtil.performDPLTest(query, testFile, ds -> {
+            Assertions.assertEquals(schema, ds.schema().toString());
+            Row r = ds.select("quotedField").distinct().first();
+
+            Assertions.assertEquals("string", r.getAs(0));
+        });
+    }
+
+    @Test
+    @DisabledIfSystemProperty(
+            named = "skipSparkTest",
+            matches = "true"
+    )
+    public void evalFieldWithSingleQuotes() {
+        String query = "index=index_A | eval 'quotedField' = \"string\"";
+        String schema = "StructType(StructField(_raw,StringType,true),StructField(_time,TimestampType,true),StructField(host,StringType,true),StructField(id,LongType,true),StructField(index,StringType,true),StructField(offset,LongType,true),StructField(partition,StringType,true),StructField(source,StringType,true),StructField(sourcetype,StringType,true),StructField(quotedField,StringType,false))";
+        String testFile = "src/test/resources/spath/spathTransformationTest_numeric2*.jsonl";
+
+        streamingTestUtil.performDPLTest(query, testFile, ds -> {
+            Assertions.assertEquals(schema, ds.schema().toString());
+            Row r = ds.select("quotedField").distinct().first();
+
+            Assertions.assertEquals("string", r.getAs(0));
+        });
+    }
 }