Skip to content

Commit

Permalink
#405 Fix the root level redefine issue
Browse files Browse the repository at this point in the history
  • Loading branch information
yruslan committed Jul 28, 2021
1 parent fd8ce46 commit f1ed91b
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,9 @@ object RecordExtractors {

val records = for (record <- ast.children) yield {
val (size, values) = getGroupValues(nextOffset, record.asInstanceOf[Group])
nextOffset += size
if (!record.isRedefined) {
nextOffset += size
}
values
}

Expand Down
2 changes: 1 addition & 1 deletion data/test12_expected/test12.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"RECORD_A":{"NAME":{"SHORT_NAME":{"NAME_CHAR_1":"F","SHORT_NAME_REST":"IRST NAME"},"REST":"LAST NAME"},"ACCOUNT_NO":{"FIRST_6":123456,"ACCOUNT_MIDDLE":0,"LAST_4":1234}},"RECORD_B":{"NAME":""}}
{"RECORD_A":{"NAME":{"SHORT_NAME":{"NAME_CHAR_1":"F","SHORT_NAME_REST":"IRST NAME"},"REST":"LAST NAME"},"ACCOUNT_NO":{"FIRST_6":123456,"ACCOUNT_MIDDLE":0,"LAST_4":1234}},"RECORD_B":{"NAME":"FIRST NAME LAST NAME","ACCOUNT_NO":1234560000001234}}
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,7 @@ class Test14RootLevelRedefines extends WordSpec with SparkTestBase with BinaryFi
.load(tmpFileName)

// This is an error. Should be:
//val expected = """[{"A":"1","B":"1"},{"A":"2","B":"2"},{"A":"3","B":"3"},{"A":"4","B":"4"}]"""
val expected = """[{"A":"1","B":""},{"A":"2","B":""},{"A":"3","B":""},{"A":"4","B":""}]"""
val expected = """[{"A":"1","B":"1"},{"A":"2","B":"2"},{"A":"3","B":"3"},{"A":"4","B":"4"}]"""

val count = df.count()
val actual = df.toJSON.collect().mkString("[", ",", "]")
Expand Down Expand Up @@ -154,8 +153,7 @@ class Test14RootLevelRedefines extends WordSpec with SparkTestBase with BinaryFi

val actualLayout = CopybookParser.parseTree(copybook2Roots).generateRecordLayoutPositions()
// This is an error. Should be:
//val expected = """[{"A":"1","B":"1"},{"A":"2","B":"2"},{"A":"3","B":"3"},{"A":"4","B":"4"}]"""
val expected = """[{"A":"1","B":""},{"A":"2","B":""},{"A":"3","B":""},{"A":"4","B":""}]"""
val expected = """[{"A":"1","B":"1"},{"A":"2","B":"2"},{"A":"3","B":"3"},{"A":"4","B":"4"}]"""

val count = df.count()
val actual = df.toJSON.collect().mkString("[", ",", "]")
Expand Down Expand Up @@ -188,9 +186,7 @@ class Test14RootLevelRedefines extends WordSpec with SparkTestBase with BinaryFi

val actualLayout = CopybookParser.parseTree(copybook2Roots).generateRecordLayoutPositions()

// This is an error. Should be:
//val expected = """[{"ENTITY1":{"A":"1"},"ENTITY2":{"B":"1"}},{"ENTITY1":{"A":"2"},"ENTITY2":{"B":"2"}},{"ENTITY1":{"A":"3"},"ENTITY2":{"B":"3"}},{"ENTITY1":{"A":"4"},"ENTITY2":{"B":"4"}}]"""
val expected = """[{"ENTITY1":{"A":"1"},"ENTITY2":{"B":""}},{"ENTITY1":{"A":"2"},"ENTITY2":{"B":""}},{"ENTITY1":{"A":"3"},"ENTITY2":{"B":""}},{"ENTITY1":{"A":"4"},"ENTITY2":{"B":""}}]"""
val expected = """[{"ENTITY1":{"A":"1"},"ENTITY2":{"B":"1"}},{"ENTITY1":{"A":"2"},"ENTITY2":{"B":"2"}},{"ENTITY1":{"A":"3"},"ENTITY2":{"B":"3"}},{"ENTITY1":{"A":"4"},"ENTITY2":{"B":"4"}}]"""

val count = df.count()
val actual = df.toJSON.collect().mkString("[", ",", "]")
Expand Down

0 comments on commit f1ed91b

Please sign in to comment.