Skip to content

Commit

Permalink
#423 Update test suites to handle new field.
Browse files Browse the repository at this point in the history
  • Loading branch information
@yruslan
yruslan committed Apr 25, 2022
1 parent 902cf61 commit 6b0dbce
Show file tree
Hide file tree
Showing 7 changed files with 39 additions and 25 deletions.
1 change: 1 addition & 0 deletions ...c/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test11CustomRDWParser.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ class Test11CustomRDWParser extends FunSuite with SparkTestBase {
.option("record_header_parser", "za.co.absa.cobrix.spark.cobol.source.utils.Test10CustomRDWParser")
.option("rhp_additional_info", "rhp info")
.load(inputDataPath)
.drop("Record_Byte_Length")

val expectedSchema = Files.readAllLines(Paths.get(expectedSchemaPath), StandardCharsets.ISO_8859_1).toArray.mkString("\n")
val actualSchema = df.schema.json
Expand Down
6 changes: 3 additions & 3 deletions ...est/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test14RdwAdjustmentsSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class Test14RdwAdjustmentsSpec extends FunSuite with SparkTestBase {

private val exampleName = "Test4(multisegment,ascii)"
private val inputCopybookPath = "file://../data/test14_copybook.cob"
private val inpudDataPath = "../data/test14_data"
private val inputDataPath = "../data/test14_data"
private val expectedSchemaPath = "../data/test14_expected/test14_schema.json"
private val actualSchemaPath = "../data/test14_expected/test14_schema_actual.json"
private val expectedResultsPath = "../data/test14_expected/test14.txt"
Expand All @@ -51,7 +51,7 @@ class Test14RdwAdjustmentsSpec extends FunSuite with SparkTestBase {
.option("redefine_segment_id_map:0", "STATIC-DETAILS => C,D")
.option("redefine-segment-id-map:1", "CONTACTS => P")
.option("is_rdw_part_of_record_length", "true")
.load(inpudDataPath)
.load(inputDataPath)

val expectedSchema = Files.readAllLines(Paths.get(expectedSchemaPath), StandardCharsets.ISO_8859_1).toArray.mkString("\n")
val actualSchema = SparkUtils.prettyJSON(df.schema.json)
Expand Down Expand Up @@ -94,7 +94,7 @@ class Test14RdwAdjustmentsSpec extends FunSuite with SparkTestBase {
.option("redefine_segment_id_map:0", "STATIC-DETAILS => C,D")
.option("redefine-segment-id-map:1", "CONTACTS => P")
.option("rdw_adjustment", "-4")
.load(inpudDataPath)
.load(inputDataPath)

val expectedSchema = Files.readAllLines(Paths.get(expectedSchemaPath), StandardCharsets.ISO_8859_1).toArray.mkString("\n")
val actualSchema = SparkUtils.prettyJSON(df.schema.json)
Expand Down
1 change: 1 addition & 0 deletions .../scala/za/co/absa/cobrix/spark/cobol/source/integration/Test19DisplayNumParsingSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class Test19DisplayNumParsingSpec extends WordSpec with SparkTestBase with Cobol
.option("generate_record_id", "true")
.option("schema_retention_policy", "collapse_root")
.load(inputDataPath)
.drop("Record_Byte_Length")

//testSchema(df, actualSchemaPath, expectedSchemaPath)

Expand Down
8 changes: 4 additions & 4 deletions ...test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test20InputFileNameSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ class Test20InputFileNameSpec extends WordSpec with SparkTestBase {
.option("generate_record_id", "true")
.load(inputDataPath)

assert(df.schema.fields(2).name == "F")
assert(df.schema.fields(3).name == "F")

val fileName = getInputFileNameVarLen(df)

Expand All @@ -158,7 +158,7 @@ class Test20InputFileNameSpec extends WordSpec with SparkTestBase {
.option("generate_record_id", "true")
.load(inputDataPath)

assert(df.schema.fields(2).name == "F")
assert(df.schema.fields(3).name == "F")

val fileName = getInputFileNameVarLen(df)

Expand Down Expand Up @@ -218,7 +218,7 @@ class Test20InputFileNameSpec extends WordSpec with SparkTestBase {
val df = getDataFrameReader(getHierarchicalDf, true, false)
.load(inputDataPath)

assert(df.schema.fields(2).name == "F")
assert(df.schema.fields(3).name == "F")

val fileName = getInputFileNameVarLen(df)

Expand All @@ -229,7 +229,7 @@ class Test20InputFileNameSpec extends WordSpec with SparkTestBase {
val df = getDataFrameReader(getHierarchicalDf, true, true)
.load(inputDataPath)

assert(df.schema.fields(2).name == "F")
assert(df.schema.fields(3).name == "F")

val fileName = getInputFileNameVarLen(df)

Expand Down
1 change: 1 addition & 0 deletions ...scala/za/co/absa/cobrix/spark/cobol/source/integration/Test22HierarchicalOccursSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ class Test22HierarchicalOccursSpec extends FunSuite with SparkTestBase with Bina
.option("redefine-segment-id-map:2", "SEG2 => 2")
.option("segment-children:1", "SEG1 => SEG2")
.load(tmpFileName)
.drop("Record_Byte_Length")

val expected =
"""[ {
Expand Down
36 changes: 18 additions & 18 deletions .../src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test04VarcharFields.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,16 @@ class Test04VarcharFields extends FunSuite with SparkTestBase with BinaryFileFix
.load(tmpFileName)

val expected =
"""+-------+---------+---+----------+
||File_Id|Record_Id|N |V |
|+-------+---------+---+----------+
||0 |0 |0 |1234567890|
||0 |1 |1 |2345678 |
||0 |2 |2 |2345678 |
||0 |3 |3 |123 |
||0 |4 |4 |1 |
||0 |5 |5 | |
|+-------+---------+---+----------+
"""+-------+---------+------------------+---+----------+
||File_Id|Record_Id|Record_Byte_Length|N |V |
|+-------+---------+------------------+---+----------+
||0 |0 |11 |0 |1234567890|
||0 |1 |11 |1 |2345678 |
||0 |2 |10 |2 |2345678 |
||0 |3 |4 |3 |123 |
||0 |4 |2 |4 |1 |
||0 |5 |1 |5 | |
|+-------+---------+------------------+---+----------+
|
|""".stripMargin.replace("\r\n", "\n")

Expand All @@ -117,21 +117,21 @@ class Test04VarcharFields extends FunSuite with SparkTestBase with BinaryFileFix
.load(tmpFileName)

val expected =
"""+-------+---------+---+---+
||File_Id|Record_Id|N |V |
|+-------+---------+---+---+
||0 |0 |3 |123|
||0 |1 |4 |1 |
||0 |2 |5 | |
|+-------+---------+---+---+
"""+-------+---------+------------------+---+---+
||File_Id|Record_Id|Record_Byte_Length|N |V |
|+-------+---------+------------------+---+---+
||0 |0 |4 |3 |123|
||0 |1 |2 |4 |1 |
||0 |2 |1 |5 | |
|+-------+---------+------------------+---+---+
|
|""".stripMargin.replace("\r\n", "\n")

val expectedJson =
"""{"File_Id":0,"Record_Id":0,"N":"3","V":"123"},{"File_Id":0,"Record_Id":1,"N":"4","V":"1"},{"File_Id":0,"Record_Id":2,"N":"5","V":""}"""

val actual = TestUtils.showString(df, 10)
val actualJson = df.toJSON.collect().mkString(",")
val actualJson = df.drop("Record_Byte_Length").toJSON.collect().mkString(",")

assertEqualsMultiline(actual, expected)
assertEqualsMultiline(actualJson, expectedJson)
Expand Down
11 changes: 11 additions & 0 deletions .../src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test18AsciiNulChars.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,56 +124,67 @@ class Test18AsciiNulChars extends WordSpec with SparkTestBase with BinaryFileFix
"""[ {
| "File_Id" : 0,
| "Record_Id" : 0,
| "Record_Byte_Length" : 1,
| "A" : "1",
| "B" : ""
|}, {
| "File_Id" : 0,
| "Record_Id" : 1,
| "Record_Byte_Length" : 2,
| "A" : "1",
| "B" : "2"
|}, {
| "File_Id" : 0,
| "Record_Id" : 2,
| "Record_Byte_Length" : 3,
| "A" : "1",
| "B" : "23"
|}, {
| "File_Id" : 0,
| "Record_Id" : 3,
| "Record_Byte_Length" : 4,
| "A" : "1",
| "B" : "234"
|}, {
| "File_Id" : 0,
| "Record_Id" : 4,
| "Record_Byte_Length" : 5,
| "A" : "1",
| "B" : "234"
|}, {
| "File_Id" : 0,
| "Record_Id" : 5,
| "Record_Byte_Length" : 6,
| "A" : "1",
| "B" : "234"
|}, {
| "File_Id" : 0,
| "Record_Id" : 6,
| "Record_Byte_Length" : 7,
| "A" : "1",
| "B" : "234"
|}, {
| "File_Id" : 0,
| "Record_Id" : 7,
| "Record_Byte_Length" : 8,
| "A" : "1",
| "B" : "234"
|}, {
| "File_Id" : 0,
| "Record_Id" : 8,
| "Record_Byte_Length" : 9,
| "A" : "1",
| "B" : "234"
|}, {
| "File_Id" : 0,
| "Record_Id" : 9,
| "Record_Byte_Length" : 4,
| "A" : "1",
| "B" : "234"
|}, {
| "File_Id" : 0,
| "Record_Id" : 10,
| "Record_Byte_Length" : 4,
| "A" : "5",
| "B" : "678"
|} ]
Expand Down

0 comments on commit 6b0dbce

Please sign in to comment.