diff --git a/src/main/java/com/teragrep/pth10/steps/eval/EvalStep.java b/src/main/java/com/teragrep/pth10/steps/eval/EvalStep.java index d871f872..eeef8f0d 100644 --- a/src/main/java/com/teragrep/pth10/steps/eval/EvalStep.java +++ b/src/main/java/com/teragrep/pth10/steps/eval/EvalStep.java @@ -45,6 +45,8 @@ */ package com.teragrep.pth10.steps.eval; +import com.teragrep.pth10.ast.TextString; +import com.teragrep.pth10.ast.UnquotedText; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.functions; @@ -78,7 +80,7 @@ public Dataset get(Dataset dataset) { } // perform eval and long->ts - ds = ds.withColumn(leftSide, rightSide); + ds = ds.withColumn(new UnquotedText(new TextString(leftSide)).read(), rightSide); if (timeColumnExists) { ds = ds.withColumn("_time", functions.from_unixtime(functions.col("_time")).cast(DataTypes.TimestampType)); } diff --git a/src/test/java/com/teragrep/pth10/RenameTransformationTest.java b/src/test/java/com/teragrep/pth10/RenameTransformationTest.java index 315eba9c..a6c4a4d3 100644 --- a/src/test/java/com/teragrep/pth10/RenameTransformationTest.java +++ b/src/test/java/com/teragrep/pth10/RenameTransformationTest.java @@ -119,4 +119,35 @@ public void testRenameMultipleFields() { } ); } + + @Test + @DisabledIfSystemProperty( + named = "skipSparkTest", + matches = "true" + ) + public void rename_DoubleQuotes_test() { + streamingTestUtil + .performDPLTest( + "| makeresults count=1 | eval \"a\" = \"something\" | rename \"a\" as \"b\"", testFile, ds -> { + Assertions + .assertEquals("[_time, b]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + } + ); + } + + @Test + @DisabledIfSystemProperty( + named = "skipSparkTest", + matches = "true" + ) + public void rename_SingleQuotes_test() { + streamingTestUtil + .performDPLTest( + "| makeresults count=1 | eval 'abc(def)' = \"xyz\" | rename 'abc(def)' as '(foo)bar'", testFile, + ds -> { + Assertions + .assertEquals("[_time, (foo)bar]", Arrays.toString(ds.columns()), "Batch handler dataset contained an unexpected column arrangement !"); + } + ); + } } diff --git a/src/test/java/com/teragrep/pth10/evalTest.java b/src/test/java/com/teragrep/pth10/evalTest.java index 67843fd0..82076c2a 100644 --- a/src/test/java/com/teragrep/pth10/evalTest.java +++ b/src/test/java/com/teragrep/pth10/evalTest.java @@ -3589,4 +3589,40 @@ public void evalAfterSpath_ComparisonTest() { Assertions.assertEquals(expected, a); }); } + + @Test + @DisabledIfSystemProperty( + named = "skipSparkTest", + matches = "true" + ) + public void evalFieldWithDoubleQuotes() { + String query = "index=index_A | eval \"quotedField\" = \"string\""; + String schema = "StructType(StructField(_raw,StringType,true),StructField(_time,TimestampType,true),StructField(host,StringType,true),StructField(id,LongType,true),StructField(index,StringType,true),StructField(offset,LongType,true),StructField(partition,StringType,true),StructField(source,StringType,true),StructField(sourcetype,StringType,true),StructField(quotedField,StringType,false))"; + String testFile = "src/test/resources/spath/spathTransformationTest_numeric2*.jsonl"; + + streamingTestUtil.performDPLTest(query, testFile, ds -> { + Assertions.assertEquals(schema, ds.schema().toString()); + Row r = ds.select("quotedField").distinct().first(); + + Assertions.assertEquals("string", r.getAs(0)); + }); + } + + @Test + @DisabledIfSystemProperty( + named = "skipSparkTest", + matches = "true" + ) + public void evalFieldWithSingleQuotes() { + String query = "index=index_A | eval 'quotedField' = \"string\""; + String schema = "StructType(StructField(_raw,StringType,true),StructField(_time,TimestampType,true),StructField(host,StringType,true),StructField(id,LongType,true),StructField(index,StringType,true),StructField(offset,LongType,true),StructField(partition,StringType,true),StructField(source,StringType,true),StructField(sourcetype,StringType,true),StructField(quotedField,StringType,false))"; + String testFile = "src/test/resources/spath/spathTransformationTest_numeric2*.jsonl"; + + streamingTestUtil.performDPLTest(query, testFile, ds -> { + Assertions.assertEquals(schema, ds.schema().toString()); + Row r = ds.select("quotedField").distinct().first(); + + Assertions.assertEquals("string", r.getAs(0)); + }); + } }