From 261ed7094792bfcfe04d54c48f39afa2100220a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A3=B8=E5=A5=94=E4=B8=B6=E5=B0=8F=E9=A6=92=E5=A4=B4?= Date: Fri, 24 May 2024 10:33:42 +0800 Subject: [PATCH] [BugFix] Fix the crash caused by JniScanner (#44903) Signed-off-by: changxin (cherry picked from commit b8cbc29f099bdd89876c1f60e717419939a82cca) --- .../hive/reader/TestHiveScanner.java | 46 ++++++++++++++++++ .../test_complex_type/complex_type_test.avro | Bin 0 -> 734366 bytes .../jni/connector/OffHeapColumnVector.java | 29 ++++++++--- 3 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 java-extensions/hive-reader/src/test/resources/test_complex_type/complex_type_test.avro diff --git a/java-extensions/hive-reader/src/test/java/com/starrocks/hive/reader/TestHiveScanner.java b/java-extensions/hive-reader/src/test/java/com/starrocks/hive/reader/TestHiveScanner.java index 0e0b06b844c6d..818fdb19f5ef6 100644 --- a/java-extensions/hive-reader/src/test/java/com/starrocks/hive/reader/TestHiveScanner.java +++ b/java-extensions/hive-reader/src/test/java/com/starrocks/hive/reader/TestHiveScanner.java @@ -44,6 +44,45 @@ Map createScanTestParams() { return params; } + Map createComplexTypeScanTestParams() { + Map params = new HashMap<>(); + URL resource = TestHiveScanner.class.getResource("/test_complex_type"); + String basePath = resource.getPath().toString(); + String filePath = basePath + "/complex_type_test.avro"; + File file = new File(filePath); + params.put("data_file_path", filePath); + params.put("block_offset", "0"); + params.put("block_length", String.valueOf(file.length())); + params.put("hive_column_names", + "id,array_col,map_col,struct_col"); + params.put("hive_column_types", + "string#array#map#struct,info:map>"); + params.put("input_format", "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat"); + params.put("serde", "org.apache.hadoop.hive.serde2.avro.AvroSerDe"); + params.put("required_fields", "id,array_col,map_col,struct_col"); + return params; + } + + String runComplexTypeScanOnParams(Map params) throws Exception { + HiveScanner scanner = new HiveScanner(4096, params); + System.out.println(scanner.toString()); + scanner.open(); + StringBuilder sb = new StringBuilder(); + while (true) { + scanner.getNextOffHeapChunk(); + OffHeapTable table = scanner.getOffHeapTable(); + if (table.getNumRows() == 0) { + break; + } + table.show(4096); + sb.append(table.dump(4096)); + table.checkTableMeta(true); + table.close(); + } + scanner.close(); + return sb.toString(); + } + String runScanOnParams(Map params) throws IOException { HiveScanner scanner = new HiveScanner(4096, params); System.out.println(scanner.toString()); @@ -70,4 +109,11 @@ public void c1DoScanTestOnPrimitiveType() throws IOException { Map params = createScanTestParams(); runScanOnParams(params); } + + @Test + public void complexTypeTest() throws Exception { + Map params = createComplexTypeScanTestParams(); + // if error, NegativeArraySizeException will be throw + runComplexTypeScanOnParams(params); + } } diff --git a/java-extensions/hive-reader/src/test/resources/test_complex_type/complex_type_test.avro b/java-extensions/hive-reader/src/test/resources/test_complex_type/complex_type_test.avro new file mode 100644 index 0000000000000000000000000000000000000000..c013a43a7b0b99d8bf7d5788d9904efdd19dc0f6 GIT binary patch literal 734366 zcmeI*K~G%g6~=LTDoRv!g+^UhtSu`f5aS_a(=AdLv>T$xn!pU1Y77=&r)iXL!xvy# zsS+Om7JLBIJ!0EEeec*#2mFqC;ohsr&TpBRjGgN<&+|Xed(M0R^6lS$^T{83ulDyI zA8dWsf8P7gAAVdo{NY7^p}Vl(-`d;XUU;;y+k0O9^VZ(;7o+|^o>jm6?67}u_~w@m zUi7xcKi~U)yLZ@2|8_9!kG2mMx?lhJ_P-rgpY!^2zh2mVIU1#ZJUHAR?(Qsn^Jrna zKj^(29WHdMU-{|L&;M0#f4}#`v#q`Ar*HnpKmQ@eKazfVc-Vhl{f@Uk{HOPP{PW(6 z_kVc$f%MU@dZU;9`+j!%pL@A=`2LT-{eWNp^RrLW?^!+ax6d{G9yHdFRva_xCma{cimKyyv&A{!gd>ekFf}c2P~Zhex=z(Km2~s8y)m#x~uair|0XN7i*oX!SVIR+3eBfT*P8@B6<6Pl|pT;QS)0t@T zll_rB?h7QrG|1dCu1EHaHaeKZp<<7K>zm+>-Q#>;paFXLsr zjF+u--E+rHBc+kjNNJ=rQW`0ZltxM;rIFG|X{0n#8Yvg2G;xcJ4qJz^t8t^PYJaaNM`RfVB26ox~za^RvE6oX<=42t2{y=IHXN>~XiVI{1D zm9Ub=!aj#+4$-o{fgFGb$z zZ5Bxlwwrqv7z1Nq42*#>Fb2kG&Q}fU7}Ux7s=`nh3gZ`=;Oef8Y8( zvl3RqN>~XiVI{1@c4!*|V;BQtU<{0bF)#+kz!(@KKAmwO>_9knU=b{WMX(4K!6H}$ zi((~U5iF|2a5)AMUrc!x1}F_sW_t&Pp)eH2UGnKf%869=TcR)&hQa`!GC7%?Oim^z zlatBGDUFmyN+U%v zG*TKVjg&@8Bc+kjNR{*sWFb2lJ7#IU%V2tt91T2C@um~2xB3J~AU=b{WMR9AI)WVU4 zqbwsT427XEY@(Gn4-|u9Pz;JeF;b(o7~e+RXY*fJE3D0`r@~Me3PXxtZmSf7Vo(f< zK{1^BIrnq!=iJY^pL4(Im&o8{yo{IeGG4~Zco{F_WxR}+@v}|jC=7+6 zFs$2_Pc2alia{|b2E|C3odx}2Xjhx)(5w$wAIRFO!cZ6r!(K@_Z&3`2K`|%>#V~Sh zw#Xrx*&?$=W{b=gnJqF~B&-$Ink{mM?F>7ckSGj=p)d?Llt*V2gJMt&ia{}qL)8r1 zSqUp)C9H&%u##F<(p{ZDIXz$Byjbg84UVrj&SsA;=c-}lPE;r2DFsg@*5Gzwd!Ou& z%o155OJs>Gv6dxrzqF|HG)7?*#wd(Y7^5&oVT{5Ug=x4E)~;dYyuX+B3J~AU=b{W zMX(4K!J=3RleN`bY@bx`vAy{)@G@Rj7zm+>-QzJ-WJ`;7Kw5m8|%429t( zCgr^Y#h@4zgJMt&uRW@nHn0*_!b(^PD`6$9#2fV52(B>(#=saD17lzejDayQhO0>p zP`aAb)ugT_bv3E0NnK59eZcxaHX%_M3PWMoD=Cl8Czm+>-Qc7}b=`v5gk8YzvGMoJ^4k#$Q_upO`n7QrG|1dCu1EP_R_=)=IG?&|!>>G}HR#aicTaD2USHhXkA z7wxS#wfD)6sT@-|rgBW>n94DgV=8%!JVqWPkCDg7W8^XN73AnPEj>1qF3d05C z#jqh~=WSc%If+BgHn7#IU%U<{0bF)#+kz!>8>D_8`JU=b{W zMX(4K!6H}$i{jS&xfMk#idjZf7z#sS*j+7e9w-LIpcoW`Vx&fEG48qICae|KX4O++ zC=7)m#V@y2ia{|b2F0Kl&i$PGIrnq!=iJY^-}Gyz@G@S;%Xk?t<7K>zm+>-Q#>;p) zy%uAsy0m&Z2GbvgsTWf(+2%oEC=7*hhkA-$L@(JdiNa7A3Ilw~zm+`VQYn71j!Cg|)B<7DZ~3k`|T{sbU9bWoSmq~F%#7{OEm3$vOlr{ z&lI>?PJ#W?7=@rtP$#Gp)CuYYbxmYbNN&Pc!6(K5pMp9;ouE!o*Vy?N#=sa^TU8he zLt(fyqnx)W2F0Kl6oX>8W~XL{#!6TTD`6$9gq5%o)9&Wv%odp~GFxP}$ZV0>BC|ye zkIb7bGFz058x)4ZP#A{J%5x}+K`|%>#h@6*RBI;Jtb~=Y5>~=WSP3h!@zBO0D8|4T z7z1Nq42*#>Fb2jL4_d(@SOkk;5iEj5um~2xB3KkF;hxXxjSEkz_j28Q7-Q#>;paFW*8$S9-Y8Ba4U%Lt!WkcfFMN4itl8Pz;JeG29MSGi_ictb~=Y5>~=W zSc!`U+X${P2FAb`7z1Nq42*#>Fow&F4N$tw*k#5pGj^G=%Zy!SY<DeH2`gbGti)1t8|Std17lzejDayQ2FAb`7{kRW zE>4*~8-|zhGG4~Zco{F_WxR}+@iJb<%MnYJ7%s-Q#>;paFXLsrjF<5;UUr6k&-(y1 zQW`0ZltxM;rIFG|X{0n#8YzvGMoJ^4ks8fJ=RvQVF$`=NINMYz427XEOo^AL4HSc7 zPz;JeF&;$Xj&~gIWaU#~C=7+++^^hMDF(%$7!>0pR*dfI{K@J0`sT%2=W1|#y>T{s zbU7F8MG)iTlKkmo<3^f~SHq>mW*-*2gW<$+} znhiA@YL3{6fJLwf7QrG|1dCu1EP_R_D8;4Q0gGS}EP_R_2o}L2SOkk;QET%wt$na@ zDC1Ds2_S``FcgM;^77Fzia{|b2F0KlZlI_+HNi?)2`gbGtb~=YlID(@#fbL(vc9S? z6o$gE2wDzY6oX<=42nT9td7@g(O3y9VI{1Dm9P?4Vz0zriMS7NUu z^##^#qnZRrdeH2{5WR?AL@%Nj(TnIs^dfq}7@`-^i|9r4B6<&m7L z3PWKi4EuiNF%-q17!-qIPz)QwHG^wb!b(^PD`6$9gq65#qKz|9jDayQ2FAb`7z1Nq z42&_Jvw}si2o}L2SOkk;5iEj5uqbZLpIcG1qL^hwg`qGMhTYZj=7C~R42nT9C`M|u z7UP~PZo*n&ZB{)MhQd%7Qv7mTr5F^0Vo(f<;oQ%;pL0Lwe$M@z`%S-g3NPbjyo{Ie zGG4~Zco{F_WxR}+(`zx7s!OYvV=(<;n0hhwl5HLohQd%7cc`c6Mf8&Wk|+#?p)kOw zOim^zlatBG35pcoW`Vo(f~=WSP3g( zC5@ebX;-zYSzA>Y3PWMIG^3oiC~XivGLHxAt=Vc7#IU%U<{0bF)#+k7!O*(B3J~AU=b{WMX(4K!6H}`E8)uM>WvFe zs`ql;d>D8cFDneZjF<5;UdGFK886>LL|1ya(j$wA3PWKi40pYh_YM?;Vo(f~=WSP3g(C9K3!a~tQj7z1Nq z42*#>Fb2lJ7#PFFDK1W#J{yLY@iJb<%Xk?t<7K>zm+>-Q#>){)l^8C^VEVJLhiwl# z+dC)>g`qI+l26f#=q39tQ5XtCVSrDWoJ>w8CzF%O$>d~m({m`ijF<5;UdGFK8872y zyo{IeGG2Cueb4&zfyAovXp|^~Txk z(dAsU$BpfMvhSqt~XiVI?k`XyXhNV_*!7 zfiW-!#=saD17nQmtY8r=f<>?h7QrG|1dCu1EQ(w6=T;Q0C}tT^VJHlRVRyB>d7u~+ zgJMt&ijf+v#kl8+o3K_`n^jMRp)eGN6u;b7DF(%$7!-qIIQMhz=iJY^pL0Lwe$%g= z!pnFWFXLsrjF<5;UdGFK8874I^jeIi>eA}v7)*Z{rd~|FWSa+tp)eH29qK815xr!; zBnm@eC=Bo^latBG-Q#>;paFXLsrjF<5;UdGGLuTgWomk z4BHuYwyRVa3PWKSZYYl%Cn71qKc zSQM#AN?MSnS<>QRT}JG#&Yzr~uWw$gb*=`-*BfWEN0)O|3(|>dR3d%jn*80O#ZUG} z79oj{L`Whe5mL?Agq5%oR>DeH2`gbG4RxA&zevYgPh%A0I2&X9|5>ybH+agPGLl1b zNDj#%IV6|91&tqRU<{0bF)#+kz!(?#c-8h&9s4)uo70nN>~XiVI@s%)U~+c#;_SS!)DkFn_)9-hTXFb2kO`=Hwg-9G5{!L&H^S#=xQ=hf|NOVyn7a&-gS7u6kXPpX@G&3sOM7*2FAb`7z1Nq z42*#>FoxTqObSdNo50I>8872yyo{IeGG4~Zco{F_Wh2-3oIlq{X{0n#8YzvGMoJ^4 zkSMIA6gJMt&ia{|< z)*2ykR$_$22#FCABP2#hjF1>1X<}?bhpofbVe7DU*g9+-whmi|t;5z~>#%j$I&2-b z4m1VJHmS?&UEQ#h@4zgJMvOU&s9z-PQS%)ARMsi?z zt0UK}b14jkp)j1qmm>|upcoW`Vo(gzJEnI`@0i{(y<>XE^p5Er(>tbjOz)(=z~Yp} zsZqcpSOkk;5iEj5um~2lc5u|%2b;$*@0UI5QWy$DVHgB0KLe#06oX<=42tpV7#%ky zKAaGI?lZ`U11~X7SD9r{-uSoJhrvv`AVcEs_>Vi=;);B54sUk`_seq(#yqX_2(3wRxJ> zKG(Y{~S zR~3fBP#8x0%7KewPz;JeF(`&%;F>KOD`6$9gq5%oR>Dde3;P_RIYi6as=`nh3d8AY zId4%6ia{|b2E{OsQL{s1C9H&%uo70nN?3`}zBVqA!5A0=V_*!7fiW-!#=scki|Swz zEP_R_2o}L2SOkk;5iGj>UIVlEX7RI(s4x_U!my-K-aJqYia{|b2F0+}RWoj2C9H&% zuo70nN?3^v;Wm+QDQZtLx~sV~@WZf4Ld zzFGWi+MqBLhQhFSjOW+kkIm9P?4!b(_4b7`O1BC|zVUsV_i zLtz*?D+ex$K`|%>#h@6*RBN_qtb~=Y5>~=WSP3h!@nGZO!`yi2uFjvFp096StaYvi z$JZNYvqzV6(cW2Zd!LL*5h)@?q=*!eB2q+(NIy&>1)sx-K|;Jd@h~7h#E1A0AL2uN zh!620KE!v;j%#*Yv*Vf_*X+1vC-nuc*@DeH z2`gbGZa8k^3>0Hv42*#>Fb2lJ7#IU%jOVOi5iEj5um~2xB3J~AU=b{eTl42u6s;&` z8Bt*<425BLwY+(t7!-qIPz;KZ8m+~+=Zc%KR#=-=Plcf{6owSP+*Tg5Gj~gfk#h@4zgJPs;#_dqU(3M2&LV zW2(kfjj0+_HKy9cWTp;VhpofbVe7DU*g9+-whmi|t;5z~>#%j$I&2-bMNkVz7LKwB ziNa7A3d1H^d35%{DMoj7{^az0ee+_ib2T`=-uQR?_g}xQE|2R(6_+JMPXF@_-V@Q@ zC;KB4Yhq2TKVq?_%jrtbr!fi%t^`+tE5ViEN^m8(4Yz0-DU6|!(nx8fQUK;H3f`j7 z%niAc&xzh0V_*!7fiW-!#;7@z&PrGbD`6$9gq5%o*9E6Fb2lJ z7#L$bfd`9V5iEj5um~2xB3J~AU{S1uC5`HZ+D{@$;?0MFm+`W~z{_|UFXLsrjF<89 zEkraWZc03hhzdhtC=A!Qm-h}7gJMt&ia{~F_@QRnz)DyND`6$9gq5%o?+|JuxW*V5 z17lzejDayQ2FAb`Zn-u<>6UA^T)XAkE!S?jcFVQ(0qX