diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/base/SimpleComparisonBase.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/base/SimpleComparisonBase.scala new file mode 100644 index 00000000..fbfeca28 --- /dev/null +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/base/SimpleComparisonBase.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.cobrix.cobol.base + +import org.scalatest.TestSuite +import org.slf4j.Logger + +/** + * This trait extends a test suite with the method for comparing small multiline strings. + */ +trait SimpleComparisonBase extends TestSuite { + def removeWhiteSpace(s: String): String = { + s.replaceAll("[\\r\\n ]", "") + } + + def assertEqualsMultiline(actualResults: String, expectedResults: String)(implicit logger: Logger): Unit = { + if (actualResults.replaceAll("[\r\n]", "") != expectedResults.replaceAll("[\r\n]", "")) { + logger.error(s"EXPECTED:\n$expectedResults") + logger.error(s"ACTUAL:\n$actualResults") + fail("Actual data does not match the expected data (see above).") + } + } + +} diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala index c3c86d4a..cfc72eb7 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala @@ -16,13 +16,17 @@ package za.co.absa.cobrix.spark.cobol -import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} -import org.scalatest.{FunSuite, WordSpec} +import org.apache.spark.sql.types.StructType +import org.scalatest.WordSpec +import org.slf4j.{Logger, LoggerFactory} import za.co.absa.cobrix.cobol.parser.CopybookParser import za.co.absa.cobrix.cobol.reader.policies.SchemaRetentionPolicy import za.co.absa.cobrix.spark.cobol.schema.CobolSchema +import za.co.absa.cobrix.spark.cobol.source.base.SimpleComparisonBase + +class CobolSchemaSpec extends WordSpec with SimpleComparisonBase { + private implicit val logger: Logger = LoggerFactory.getLogger(this.getClass) -class CobolSchemaSpec extends WordSpec { "for simple copybooks" should { val copyBookContents: String = """ 01 RECORD. @@ -53,7 +57,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.CollapseRoot, "", false) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } "Generate record id field" in { @@ -74,7 +78,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.CollapseRoot, "", true) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } } @@ -102,7 +106,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.KeepOriginal, "", true) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } "keep original and no record id generation" in { @@ -118,7 +122,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.KeepOriginal, "", false) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } "collapse root + record id generation" in { @@ -135,7 +139,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.CollapseRoot, "", true) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } "collapse root and no record id generation" in { @@ -149,7 +153,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.CollapseRoot, "", false) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } } @@ -178,7 +182,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.KeepOriginal, "", true, 2) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } "multi-segment keep-original without record id generation" in { @@ -195,7 +199,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.KeepOriginal, "", false, 2) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } "multi-segment collapse root with record id generation" in { @@ -213,7 +217,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.CollapseRoot, "", true, 2) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } "multi-segment collapse root without record id generation" in { @@ -228,7 +232,7 @@ class CobolSchemaSpec extends WordSpec { val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.CollapseRoot, "", false, 2) val actualSchema = cobolSchema.getSparkSchema.treeString - assert(actualSchema == expectedSchema) + assertEqualsMultiline(actualSchema, expectedSchema) } } diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test18AsciiNulChars.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test18AsciiNulChars.scala index 7782ebaf..dae73b18 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test18AsciiNulChars.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test18AsciiNulChars.scala @@ -105,7 +105,7 @@ class Test18AsciiNulChars extends WordSpec with SparkTestBase with BinaryFileFix |123456789 |12345678901234567890123456789 |5678 - |""".stripMargin + |""".stripMargin.replaceAll("\r", "") "not generate redundant records" in { withTempTextFile("ascii_nul", ".dat", StandardCharsets.UTF_8, text) { tmpFileName =>