diff --git a/data/test6_expected/test6a_schema.json b/data/test6_expected/test6a_schema.json new file mode 100644 index 000000000..2deda7b42 --- /dev/null +++ b/data/test6_expected/test6a_schema.json @@ -0,0 +1,985 @@ +{ + "type" : "struct", + "fields" : [ { + "name" : "ID", + "type" : "decimal(7,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "STRING_VAL", + "type" : "string", + "nullable" : true, + "metadata" : { + "maxLength" : 10 + } + }, { + "name" : "NUM_STR_INT01", + "type" : "decimal(1,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT02", + "type" : "decimal(2,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT03", + "type" : "decimal(3,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT04", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT05", + "type" : "decimal(5,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT06", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT07", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT08", + "type" : "decimal(10,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT09", + "type" : "decimal(11,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT10", + "type" : "decimal(17,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT11", + "type" : "decimal(18,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT12", + "type" : "decimal(19,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT13", + "type" : "decimal(20,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_INT14", + "type" : "decimal(37,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT02", + "type" : "decimal(2,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT03", + "type" : "decimal(3,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT04", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT05", + "type" : "decimal(5,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT06", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT07", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT08", + "type" : "decimal(10,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT09", + "type" : "decimal(11,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT10", + "type" : "decimal(17,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT11", + "type" : "decimal(18,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT12", + "type" : "decimal(19,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT13", + "type" : "decimal(20,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SINT14", + "type" : "decimal(37,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC01", + "type" : "decimal(3,1)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC02", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC03", + "type" : "decimal(5,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC04", + "type" : "decimal(8,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC05", + "type" : "decimal(9,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC06", + "type" : "decimal(10,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC07", + "type" : "decimal(17,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC08", + "type" : "decimal(18,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC09", + "type" : "decimal(19,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_DEC10", + "type" : "decimal(28,10)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC01", + "type" : "decimal(3,1)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC02", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC03", + "type" : "decimal(5,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC04", + "type" : "decimal(8,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC05", + "type" : "decimal(9,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC06", + "type" : "decimal(10,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC07", + "type" : "decimal(17,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC08", + "type" : "decimal(18,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC09", + "type" : "decimal(19,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_SDEC10", + "type" : "decimal(28,10)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_EDEC03", + "type" : "decimal(5,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_EDEC04", + "type" : "decimal(8,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_EDEC05", + "type" : "decimal(9,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STR_EDEC06", + "type" : "decimal(10,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT01", + "type" : "decimal(1,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT02", + "type" : "decimal(2,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT03", + "type" : "decimal(3,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT04", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT05", + "type" : "decimal(5,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT06", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT07", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT08", + "type" : "decimal(10,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT09", + "type" : "decimal(11,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT10", + "type" : "decimal(17,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT11", + "type" : "decimal(18,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT12", + "type" : "decimal(19,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT13", + "type" : "decimal(20,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_INT14", + "type" : "decimal(37,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT01", + "type" : "decimal(1,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT02", + "type" : "decimal(2,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT03", + "type" : "decimal(3,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT04", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT05", + "type" : "decimal(5,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT06", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT07", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT08", + "type" : "decimal(10,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT09", + "type" : "decimal(11,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT10", + "type" : "decimal(17,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT11", + "type" : "decimal(18,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT12", + "type" : "decimal(19,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT13", + "type" : "decimal(20,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_SINT14", + "type" : "decimal(37,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC01", + "type" : "decimal(3,1)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC02", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC03", + "type" : "decimal(5,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC04", + "type" : "decimal(8,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC05", + "type" : "decimal(9,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC06", + "type" : "decimal(10,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC07", + "type" : "decimal(17,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC08", + "type" : "decimal(18,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC09", + "type" : "decimal(19,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BIN_DEC10", + "type" : "decimal(28,10)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC01", + "type" : "decimal(3,1)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC02", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC03", + "type" : "decimal(5,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC04", + "type" : "decimal(8,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC05", + "type" : "decimal(9,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC06", + "type" : "decimal(10,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC07", + "type" : "decimal(17,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC08", + "type" : "decimal(18,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC09", + "type" : "decimal(19,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SBIN_DEC10", + "type" : "decimal(28,10)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT01", + "type" : "decimal(1,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT02", + "type" : "decimal(2,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT03", + "type" : "decimal(3,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT04", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT05", + "type" : "decimal(5,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT06", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT07", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT08", + "type" : "decimal(10,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT09", + "type" : "decimal(11,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT10", + "type" : "decimal(17,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT11", + "type" : "decimal(18,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT12", + "type" : "decimal(19,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT13", + "type" : "decimal(20,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_INT14", + "type" : "decimal(37,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT01", + "type" : "decimal(1,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT02", + "type" : "decimal(2,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT03", + "type" : "decimal(3,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT04", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT05", + "type" : "decimal(5,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT06", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT07", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT08", + "type" : "decimal(10,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT09", + "type" : "decimal(11,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT10", + "type" : "decimal(17,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT11", + "type" : "decimal(18,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT12", + "type" : "decimal(19,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT13", + "type" : "decimal(20,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SINT14", + "type" : "decimal(37,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC01", + "type" : "decimal(3,1)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC02", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC03", + "type" : "decimal(5,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC04", + "type" : "decimal(8,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC05", + "type" : "decimal(9,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC06", + "type" : "decimal(10,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC07", + "type" : "decimal(17,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC08", + "type" : "decimal(18,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC09", + "type" : "decimal(19,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_DEC10", + "type" : "decimal(28,10)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC01", + "type" : "decimal(3,1)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC02", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC03", + "type" : "decimal(5,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC04", + "type" : "decimal(8,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC05", + "type" : "decimal(9,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC06", + "type" : "decimal(10,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC07", + "type" : "decimal(17,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC08", + "type" : "decimal(18,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC09", + "type" : "decimal(19,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_BCD_SDEC10", + "type" : "decimal(28,10)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SL_STR_INT01", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SL_STR_DEC01", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_ST_STR_INT01", + "type" : "decimal(9,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_ST_STR_DEC01", + "type" : "decimal(4,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SLI_STR_DEC01", + "type" : "decimal(7,7)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_STI_STR_DEC01", + "type" : "decimal(7,7)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "NUM_SLI_DEBUG", + "type" : "string", + "nullable" : true, + "metadata" : { + "maxLength" : 7 + } + }, { + "name" : "NUM_STI_DEBUG", + "type" : "string", + "nullable" : true, + "metadata" : { + "maxLength" : 7 + } + }, { + "name" : "FLOAT_01", + "type" : "float", + "nullable" : true, + "metadata" : { } + }, { + "name" : "DOUBLE_01", + "type" : "double", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_8_BIN", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S3_BIN", + "type" : "decimal(3,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S94COMP", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S8_BIN", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_DDC97_BIN", + "type" : "decimal(8,7)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_97COMP3", + "type" : "decimal(7,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_915COMP3", + "type" : "decimal(15,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S95COMP3", + "type" : "decimal(5,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S999DCCOMP3", + "type" : "decimal(11,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S913COMP3", + "type" : "decimal(13,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S913DCCOMP3", + "type" : "decimal(15,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S911DCC2", + "type" : "decimal(13,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S910DCC3", + "type" : "decimal(13,3)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_S03DDC", + "type" : "decimal(5,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_U03DDC", + "type" : "decimal(5,5)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPC5DDC", + "type" : "decimal(8,8)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_SPC5DDC", + "type" : "decimal(7,7)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPI5DDC", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_SPI5DDC", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPC5DISP", + "type" : "decimal(8,8)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPI5DISP", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPC1BIN", + "type" : "decimal(4,4)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPI1BIN", + "type" : "decimal(4,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPC3BIN", + "type" : "decimal(6,6)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPI3BIN", + "type" : "decimal(6,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPC5BIN", + "type" : "decimal(8,8)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPI5BIN", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPC10BIN", + "type" : "decimal(13,13)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "COMMON_UPI10BIN", + "type" : "decimal(13,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "EX_NUM_INT01", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "EX_NUM_INT02", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "EX_NUM_INT03", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "EX_NUM_INT04", + "type" : "decimal(8,0)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "EX_NUM_DEC01", + "type" : "decimal(8,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "EX_NUM_DEC02", + "type" : "decimal(8,2)", + "nullable" : true, + "metadata" : { } + }, { + "name" : "EX_NUM_DEC03", + "type" : "decimal(8,2)", + "nullable" : true, + "metadata" : { } + } ] +} \ No newline at end of file diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test38StrictIntegralPrecisionSpec.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test38StrictIntegralPrecisionSpec.scala index fff54f629..1f94c38b0 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test38StrictIntegralPrecisionSpec.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test38StrictIntegralPrecisionSpec.scala @@ -114,6 +114,47 @@ class Test38StrictIntegralPrecisionSpec extends AnyWordSpec with SparkTestBase w } } + "be decoded as decimals with record id generation" in { + withTempBinFile("strict_integral_precision", ".tmp", dataSimple) { tempFile => + val expectedSchema = + """root + | |-- File_Id: integer (nullable = false) + | |-- Record_Id: long (nullable = false) + | |-- Record_Byte_Length: integer (nullable = false) + | |-- SEG_ID: string (nullable = true) + | |-- SEG1: struct (nullable = true) + | | |-- NUM1: decimal(2,0) (nullable = true) + | |-- SEG2: struct (nullable = true) + | | |-- NUM2: decimal(9,0) (nullable = true) + | |-- SEG3: struct (nullable = true) + | | |-- NUM3: decimal(15,0) (nullable = true)""".stripMargin + + val expectedData = """{"File_Id":0,"Record_Id":0,"Record_Byte_Length":4,"SEG_ID":"A","SEG1":{"NUM1":12}},{"File_Id":0,"Record_Id":1,"Record_Byte_Length":10,"SEG_ID":"B","SEG2":{"NUM2":123456789}},{"File_Id":0,"Record_Id":2,"Record_Byte_Length":16,"SEG_ID":"C","SEG3":{"NUM3":123456789012345}}""".stripMargin + + val df = spark.read + .format("cobol") + .option("copybook_contents", copybook) + .option("record_format", "F") + .option("record_length_field", "SEG-ID") + .option("segment_field", "SEG-ID") + .option("input_split_records", "2") + .option("pedantic", "true") + .option("record_length_map", """{"A":4,"B":10,"C":16}""") + .option("redefine_segment_id_map:0", "SEG1 => A") + .option("redefine_segment_id_map:1", "SEG2 => B") + .option("redefine_segment_id_map:2", "SEG3 => C") + .option("generate_record_id", "true") + .option("strict_integral_precision", "true") + .load(tempFile) + + val actualSchema = df.schema.treeString + val actualData = df.orderBy("SEG_ID").toJSON.collect().mkString(",") + + compareText(actualSchema, expectedSchema) + assert(actualData == expectedData) + } + } + "be decoded as decimals for hierarchical files" in { withTempBinFile("strict_integral_precision", ".tmp", dataSimple) { tempFile => val expectedSchema = diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test6TypeVarietySpec.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test6TypeVarietySpec.scala index 16fc428be..2b61d01e5 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test6TypeVarietySpec.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test6TypeVarietySpec.scala @@ -65,6 +65,7 @@ class Test6TypeVarietySpec extends AnyFunSuite with SparkTestBase { .option("schema_retention_policy", "collapse_root") .option("floating_point_format", "IEEE754") .option("strict_sign_overpunching", "true") + .option("pedantic", "true") .load(inpudDataPath) // This is to print the actual output @@ -93,7 +94,63 @@ class Test6TypeVarietySpec extends AnyFunSuite with SparkTestBase { Files.delete(Paths.get(actualResultsPathCrc)) */ - // Fill nulls with zeros so by lokking at json you can tell a field is missing. Otherwise json won't contain null fields. + // Fill nulls with zeros so by looking at json you can tell a field is missing. Otherwise json won't contain null fields. + val actualDf = df.orderBy("ID").na.fill(0).toJSON.take(100) + FileUtils.writeStringsToFile(actualDf, actualResultsPath) + val actual = Files.readAllLines(Paths.get(actualResultsPath), StandardCharsets.ISO_8859_1).asScala.toArray + + // toList is used to convert the Java list to Scala list. If it is skipped the resulting type will be Array[AnyRef] instead of Array[String] + val expected = Files.readAllLines(Paths.get(expectedResultsPath), StandardCharsets.ISO_8859_1).asScala.toArray + + if (!actual.sameElements(expected)) { + assert(false, s"The actual data doesn't match what is expected for $exampleName example. Please compare contents of $expectedResultsPath to " + + s"$actualResultsPath for details.") + } + Files.delete(Paths.get(actualResultsPath)) + } + + test(s"Integration test on Test6(type variety with strict integral precision)") { + + val expectedSchemaPath = "../data/test6_expected/test6a_schema.json" + val expectedLayoutPath = "../data/test6_expected/test6_layout.txt" + val actualSchemaPath = "../data/test6_expected/test6a_schema_actual.json" + val actualLayoutPath = "../data/test6_expected/test6a_layout_actual.txt" + val expectedResultsPath = "../data/test6_expected/test6.txt" + val actualResultsPath = "../data/test6_expected/test6a_actual.txt" + + // Comparing layout + val copybookContents = Files.readAllLines(Paths.get(inputCopybookFSPath), StandardCharsets.ISO_8859_1).toArray.mkString("\n") + val cobolSchema = CopybookParser.parseTree(copybookContents) + val actualLayout = cobolSchema.generateRecordLayoutPositions() + val expectedLayout = Files.readAllLines(Paths.get(expectedLayoutPath), StandardCharsets.ISO_8859_1).toArray.mkString("\n") + + if (actualLayout != expectedLayout) { + FileUtils.writeStringToFile(actualLayout, actualLayoutPath) + assert(false, s"The actual layout doesn't match what is expected for $exampleName example. Please compare contents of $expectedLayoutPath to " + + s"$actualLayoutPath for details.") + } + + val df = spark + .read + .format("cobol") + .option("copybook", inputCopybookPath) + .option("schema_retention_policy", "collapse_root") + .option("floating_point_format", "IEEE754") + .option("strict_sign_overpunching", "true") + .option("strict_integral_precision", "true") + .option("pedantic", "true") + .load(inpudDataPath) + + val expectedSchema = Files.readAllLines(Paths.get(expectedSchemaPath), StandardCharsets.ISO_8859_1).toArray.mkString("\n") + val actualSchema = SparkUtils.prettyJSON(df.schema.json) + + if (actualSchema != expectedSchema) { + FileUtils.writeStringToFile(actualSchema, actualSchemaPath) + assert(false, s"The actual schema doesn't match what is expected for $exampleName example. Please compare contents of $expectedSchemaPath to " + + s"$actualSchemaPath for details.") + } + + // Fill nulls with zeros so by looking at json you can tell a field is missing. Otherwise json won't contain null fields. val actualDf = df.orderBy("ID").na.fill(0).toJSON.take(100) FileUtils.writeStringsToFile(actualDf, actualResultsPath) val actual = Files.readAllLines(Paths.get(actualResultsPath), StandardCharsets.ISO_8859_1).asScala.toArray