From 334e852d0507523b6c4ee7b423e2951330218adb Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Dec 2024 01:00:27 +0100 Subject: [PATCH] Add LIBERTIFF driver: native thread-safe read-only GeoTIFF reader This driver is based on the libertiff library for IFD and tag parsing. It uses in a very controlled (and isolated) way internal libtiff LZW, PackBits and LERC codecs (even if using external libtiff for the GTiff driver). For Deflate/GZip, LZMA, ZSTD, it defers to GDAL cpl_compressor.h API. For JPEG, WEBP, JPEGXL, it uses the corresponding GDAL drivers to open strip/tile blobs as temporary datasets. Note that at current time the driver isn't necessarily optimized to minimize the number of network requests for COG layout. The driver supports most TIFF formulations, but not necessarily the most exotic ones that the GTiff libtiff based might support. In particular it does not support non-power-of-two BitsPerSample value (could be added with extra coding). On the plus side, JPEGXL is supported as soon as the JPEGXL driver is available at runtime (contrary to the GTiff driver which requires using internal libtiff since the tif_jxl.c codec hasn't been upstreamed to libtiff yet). As noted in the documentation, no side-car file at all are currently used by this driver. --- .../expected_gdalinfo_formats.txt | 1 + ...indows_conda_expected_gdalinfo_formats.txt | 1 + alg/gdal_interpolateatpoint.h | 13 +- .../gcore/data/gtiff/byte_coord_epoch.tif | Bin 0 -> 1128 bytes .../gcore/data/gtiff/cint32_big_endian.tif | Bin 0 -> 3558 bytes .../gcore/data/gtiff/int16_big_endian.tif | Bin 0 -> 1158 bytes autotest/gcore/data/gtiff/miniswhite.tif | Bin 0 -> 147 bytes ...bsmall_int16_bigendian_lzw_predictor_2.tif | Bin 0 -> 7262 bytes .../gcore/data/gtiff/sparse_nodata_one.tif | Bin 0 -> 158 bytes .../gcore/data/gtiff/sparse_tiled_contig.tif | Bin 0 -> 206 bytes .../data/gtiff/sparse_tiled_separate.tif | Bin 0 -> 254 bytes autotest/gcore/libertiff.py | 739 ++++ autotest/pymod/gdaltest.py | 8 + doc/source/drivers/raster/gtiff.rst | 3 + doc/source/drivers/raster/index.rst | 1 + doc/source/drivers/raster/libertiff.rst | 36 + frmts/CMakeLists.txt | 1 + frmts/drivers.ini | 1 + frmts/gdalallregister.cpp | 4 + frmts/gtiff/libtiff/tif_lerc.c | 54 +- frmts/gtiff/libtiff/tif_lzw.c | 11 +- frmts/gtiff/libtiff/tif_packbits.c | 8 +- frmts/libertiff/CMakeLists.txt | 46 + frmts/libertiff/libertiffdataset.cpp | 3258 +++++++++++++++++ frmts/libertiff/libtiff_codecs.h | 127 + gcore/gdal_frmts.h | 1 + gcore/gdal_priv.h | 12 + gcore/gdalallvalidmaskband.cpp | 27 + .../validate_cloud_optimized_geotiff.py | 2 +- third_party/libertiff/libertiff.hpp | 96 +- 30 files changed, 4412 insertions(+), 38 deletions(-) create mode 100644 autotest/gcore/data/gtiff/byte_coord_epoch.tif create mode 100644 autotest/gcore/data/gtiff/cint32_big_endian.tif create mode 100644 autotest/gcore/data/gtiff/int16_big_endian.tif create mode 100644 autotest/gcore/data/gtiff/miniswhite.tif create mode 100644 autotest/gcore/data/gtiff/rgbsmall_int16_bigendian_lzw_predictor_2.tif create mode 100644 autotest/gcore/data/gtiff/sparse_nodata_one.tif create mode 100644 autotest/gcore/data/gtiff/sparse_tiled_contig.tif create mode 100644 autotest/gcore/data/gtiff/sparse_tiled_separate.tif create mode 100755 autotest/gcore/libertiff.py create mode 100644 doc/source/drivers/raster/libertiff.rst create mode 100644 frmts/libertiff/CMakeLists.txt create mode 100644 frmts/libertiff/libertiffdataset.cpp create mode 100644 frmts/libertiff/libtiff_codecs.h diff --git a/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt b/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt index 396d0730f97e..db8d5841a8e3 100644 --- a/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt +++ b/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt @@ -5,6 +5,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda SNAP_TIFF -raster- (rov): Sentinel Application Processing GeoTIFF GTiff -raster- (rw+vs): GeoTIFF (*.tif, *.tiff) COG -raster- (wv): Cloud optimized GeoTIFF generator (*.tif, *.tiff) + LIBERTIFF -raster- (rov): GeoTIFF (using LIBERTIFF library) (*.tif, *.tiff) NITF -raster- (rw+vs): National Imagery Transmission Format (*.ntf) RPFTOC -raster- (rovs): Raster Product Format TOC format (*.toc) ECRGTOC -raster- (rovs): ECRG TOC format (*.xml) diff --git a/.github/workflows/windows_conda_expected_gdalinfo_formats.txt b/.github/workflows/windows_conda_expected_gdalinfo_formats.txt index f37c96eec515..5b0cb5089e3d 100644 --- a/.github/workflows/windows_conda_expected_gdalinfo_formats.txt +++ b/.github/workflows/windows_conda_expected_gdalinfo_formats.txt @@ -5,6 +5,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda SNAP_TIFF -raster- (rov): Sentinel Application Processing GeoTIFF GTiff -raster- (rw+vs): GeoTIFF (*.tif, *.tiff) COG -raster- (wv): Cloud optimized GeoTIFF generator (*.tif, *.tiff) + LIBERTIFF -raster- (rov): GeoTIFF (using LIBERTIFF library) (*.tif, *.tiff) NITF -raster- (rw+vs): National Imagery Transmission Format (*.ntf) RPFTOC -raster- (rovs): Raster Product Format TOC format (*.toc) ECRGTOC -raster- (rovs): ECRG TOC format (*.xml) diff --git a/alg/gdal_interpolateatpoint.h b/alg/gdal_interpolateatpoint.h index e92deaf7683e..735265dea43f 100644 --- a/alg/gdal_interpolateatpoint.h +++ b/alg/gdal_interpolateatpoint.h @@ -28,17 +28,18 @@ using DoublePointsCache = lru11::Cache>>; -class GDALDoublePointsCache +class CPL_DLL GDALDoublePointsCache { public: std::unique_ptr cache{}; }; -bool GDALInterpolateAtPoint(GDALRasterBand *pBand, - GDALRIOResampleAlg eResampleAlg, - std::unique_ptr &cache, - const double dfXIn, const double dfYIn, - double *pdfOutputReal, double *pdfOutputImag); +bool CPL_DLL GDALInterpolateAtPoint(GDALRasterBand *pBand, + GDALRIOResampleAlg eResampleAlg, + std::unique_ptr &cache, + const double dfXIn, const double dfYIn, + double *pdfOutputReal, + double *pdfOutputImag); /*! @endcond */ diff --git a/autotest/gcore/data/gtiff/byte_coord_epoch.tif b/autotest/gcore/data/gtiff/byte_coord_epoch.tif new file mode 100644 index 0000000000000000000000000000000000000000..89b20114959f3f96b8d6c786904f33affcdcbc24 GIT binary patch literal 1128 zcma)4&ubGw6rTJb6bVA1RgfN*wii9LNoS z22W}Xr$^YaG9&DGxiLJQ$|k+v(n8FRf z0at-P_A_D;cpvz*pHF6e0zT`f#T@dVfnR+{ivZf+@$OdjLCxQ4^p3jEx<}phYo1pP zj(P{)VfUbSf4R^OSJSSl^uE3K_m3Tt=Q%9Zk!5fTMzbbtMOG1t96B)~(pE>BmEusJ z$Fa3!(MKFg*K(o0#MT&wI)Uzn#2O68GSFhkODzWyewY%ATC+sXO*TdviIEHsC~BO+ zE$3BP6ikei9zF&ZsiR_r<)+0}x{6fiv4WH|DmK=gTSA?sfdSCqO_>hlJFtthIAf9{ z8*0c9hD+&8b~K&G#ze#&Ngd>%fXfL7Z~(XZAq<<4BN&c<{<>ggwB;zWxJrdR7=$(q M!vVV+cZ;O@4G;H4=Kufz literal 0 HcmV?d00001 diff --git a/autotest/gcore/data/gtiff/cint32_big_endian.tif b/autotest/gcore/data/gtiff/cint32_big_endian.tif new file mode 100644 index 0000000000000000000000000000000000000000..ed3e7b326a64b7de101f1edc0bfa25f0e731ffd4 GIT binary patch literal 3558 zcma*pF>6#&5C!0SH*phWQ3N-Fqj5A1^+n#h(8q7=(|;%IBhP0>xeus6#`@^$=R=GaV}1ByJhA$5 zh|e#+JYDZU9)EcDjL(;Tyomo>ryu+IfVacvjvsS|&|YzWx^&__L_hT9>B`6NDS30C zyng%3^CQm}{}4a=p=+L4pO`+LIZ;oaJf42Yk2t_n2uOo9=b2z-+Xya?SoH`ANBr! z^70v$$5W?k9zFZ3L;ZaGYTpmP8~UB5V?X`odM@zHgX;S5^wS-R-5azYU2#Nw^YGoD Ny!pqWKKIu1{{V4b;E4bL literal 0 HcmV?d00001 diff --git a/autotest/gcore/data/gtiff/int16_big_endian.tif b/autotest/gcore/data/gtiff/int16_big_endian.tif new file mode 100644 index 0000000000000000000000000000000000000000..3ba26c493cff7d9a43923059f3f47dde57ef0b49 GIT binary patch literal 1158 zcmZ{iF>6#&5QXRNhBbr`6$^{qAjiYou&Us z@$S+WIQ7>2$F||QmfoiRTKwtKhjTY$x<%Sw!<;Vtj@iwQZHosT-|LU=_vHV&{o=7g zEL*o{)*k zMd{!ZLM^?!*!Ip3o{a{tj$Rfoiu1E!arNQ-o8s_rbnU@wvrDV2wu!8^#`jvWF!oA! zQ)?Z;7v4|^U1{9f`VcfQkiOgD@SsB^jTG>p7 zKD18df3vw(t3`g)gL^7fxyqf;KOEJ##?X$=P#>Ml9S=w5YMc+YP!D5i3t7yE_-cxk z`=w&wa?eyO-N43#gAEK3&b;zmnG;NvGSLN!wK~y8e-j%kI+w-Dy)hGXL=K9X6jP=; zipXk&qBgvxS0ta?m}aoG`i_4}E%L|0M0GUgiGiMpk^d5Em~l)51u@C7XQBpH-c;>; zbbwL3+4dnn=#HSsFx5L;;RuBW~hLghD} ICkXrHKZ2p(?EnA( literal 0 HcmV?d00001 diff --git a/autotest/gcore/data/gtiff/miniswhite.tif b/autotest/gcore/data/gtiff/miniswhite.tif new file mode 100644 index 0000000000000000000000000000000000000000..7eac54b3b847526cb3877c7ba8d841930648a429 GIT binary patch literal 147 zcmebD)MDUZU|`^8U|?isU<9(j7>Uh<#AZffvq9NFEewK;EMR?;fNW7DHDX9?ai|)Q O`7%i2!ANY7SquQsoC0zH literal 0 HcmV?d00001 diff --git a/autotest/gcore/data/gtiff/rgbsmall_int16_bigendian_lzw_predictor_2.tif b/autotest/gcore/data/gtiff/rgbsmall_int16_bigendian_lzw_predictor_2.tif new file mode 100644 index 0000000000000000000000000000000000000000..a34e9d386f5213255aab9401ae6bc4d9fd60067b GIT binary patch literal 7262 zcmaKRc{r3^`2U&3zJ%C0sVMc9RAY>N4T%yOlk7W{RGRD} zA+IRe_Y_K`_|4n({=UEOU%%^I*SSB>=X}m{?&nHCKb|!MBKjXb#KL0#qnU(A{l~Kw zg&6*?`#Ar>|E~2lnji48?7w)6M*V}?S!({c>OUNUg^l8B{_y{La6PW}&-$D!oXVO5 zfWV&yc;f-!H)gTfEGx~@u-eg(!y(%apG5z!{iFZ?bgky6)9xM=J<+PY_H0PAee8dv z5Y~PH!13?Hvk(;i?-h$-3 zCLS)u#w>}ug)>4|GZIh$!a)}$%Ev(?oOF0h);@e7o=9>8yuxsh*h(z~ZM>mC2EZv0 zW8el3y!iO$N7C=%{p$dv{rJ@dxB#?Z&0_b#-U0xvzya<}1V!;OU;f#h+S|L)w5%oZ z%;kU?fD}4&U;X{v6TmQAL%0I)~?fTzm2O1oDxkC=N&hc3(0Mf&wZaM9=sGMYLuA(%jcKKl;?3e7g4H z)1^ZgSm~2HE;6otAA<)u)+7JE+=5=X;85&7>87OP%q%%_+EN%OE|zEoHR)f zk$R+?RKB%QXt{lff$F~u1X{{*@LMtqcdm~;SGY<7aK41S(1Hbty$fjGo&qjJV6$R> zxm`$jcW1g+j7*0wG)T59tO#|kV8f(sh5}YHDvUSe%IYvlk;wQufyjpV$AN8(JC940 zyrR_T;W!xzzpwY(r$MvA-Bo3S@R4$dou5pZp0VI=&D7VnTt)Fydl7XHcNT|=?VpZR zcP=01?v-AuQ|&&YNFv?Gq}HFYd&@PvDQmynT&-AX&poc(dwwW6x(gFbi&8rq?vxSB zD@@9*S*kGl`t0oLMjj-nr?uZM;dos!HnU4|poU}TrAT|(%(`)taXa7Xf#r7%mHivS z&wIT2lk<7>zwj^z<=3^MFP1fN3RLN+4I^S)MXbkN3*QBEsn%{}NY5B{iSM!A4#2pP zXNP5M=mBrbulePYiZBDKGyI$s@m4vLfy&0#Y||hCAUKoU$ob*DaKv50KV@a1@ZRN> zeSw*rl(*QKQH_+`eY_zJ5w zj-K84kSc1BP@QrVy~zuXrLf|TP~o{4eooI6p=&!m!sm05TLOna00$loVp1Wb{zjbH zof0jmh4H|?r0K8V1M76(+mg&nZ|o9{;a;ov5(r8bo{fi@W$U?FvQCxaG$SGOk1i>+ zEAGCbz{RiS9}@}?RyivA&<^{QQDGpk$R*SfXk%4cX*#Fdd#GO1O>`_gT0XC^-W1bj zeefVLjjvrA0VwHm=j;~-qLd#^!bX~r(!Fypw7aTi`iIIB64RTQ?JYknkKaM^jEEQ! zczAgwED$D|z=#q#lYE7N?~FMp(=~7{aq^qNdPtqN}D@f%}>$!h|;hF+F! zi3OIlbDEa+S@HA486qnDcH)Ygq7sf>qb{Or+lbzN&43OV93iM zE$SnFu+LG%!|xldKX0_C`9y{kQHj2NBdK;zL)iMN9c{IMRSSgbT{5i*c3vAdNAC<)6L?61v0xFv#s1+Ryb|OY47#Q33ft^6e;IJqyt>8Z zG$Y|fbB^svsInsY_sRyDuis0Q;|M2_cJkvEW4$EUyE#_LmbmorkNtyHUUotHI%v-$ zuZo}+f2~4;-Kx@Br}pqSXA|Z9q(_nyF(Klw(lIe7ybm|njtA(mMKl&R5jNQGK%K-D z1ir1`yo42Ro%4mA3L;1(My?3!kv!BXy9HuvPD>Y>s!#b3k z>8k-;Liw3~npqDM)f+japQT1OM9ey44aw~#`Z%w~H(V;7-(5n*T#6|DEoRy^{D;*W zBa&T$GO&Z~M4QpEXY#czvFw^ruXD2(dSr*O9ToY53qtg-FHb4a26CRc@+c?PF?ECZ z$ldKWLZs4}`VoI<4k7522th`n>+!|6M0WVwDK&@L*LQ9`JiDY=sw5Rl85Q+|@uy2v zswl20RD2m=mLvZcr8)b4mH4Ugd_lO=I(gp#cTjy7*sfhviQWeSOyzHi@EM&fpHSHE7s!AO|kR zP9#ja6d%rBLUVyZFH+;NV%0anQyoXj)d=PhHOkbsO}hAB z@D61dJHq4bwLaoh;&egMh<7%-Km z#?f)<8s@X`uYH&tU-AU+=L;H+{4DK`d zJ&x!kfB**&K)|$05vDAbQ&E38<^jOJ^WARNN(hV33jGpmN6uQAkyS8|*`*f@`u zR7}E%R(#uL+^twb;8Alo?bjf#GvnOXUS1~qg~^Bq?ZIm(rzhmt-@#j#qi1-ia6OnW z?rgHs*%~riwy}S@MNluc0z?VKNG ziAS&)TU7WJKm8L+!sAsxWu|Hr0?gQ%kFhDD_ujr9rq>`BQp2f{3Q@y~6>Uc7$V2)jDo)vW@5UI!KLxbg#t?pAYZz%1;NXV8{o_cEQ@rtw+g;M&ON zqIo>`ydU1U##~ninTL;NKII^(AZhQI8_md2Kh9VqxsYcL1NADuj626`>N8>vXm5L!KyF1?#>?w}c;kdHx@%|=i|0-!#gYcxER2A{>t%jdrs za=}DHk<&W*VVAXmt;|&VFC`xrdc3xCix|1Gsh`T zxAW7uaSUNPRom+gx`|VAu=f{khM7;W4@X+7p>&&r<1Gn{eU&>()f6J1RB}M#c6>H+ zmTq`J>K3W#1GG!4*!K{!_z*ksO}{Im*qwL)Err2(sG_Z?=5Lrz=5XT6s#Xh{w?X-@ zlSp?FSchAgaxm9`62oJF3H^V0h;b_}9rk;lE#@T>0l@%Uz((Oiz#Krvg?2X=dB=dOyu(yG z$G`y!+?8Y3b8q|}1vm=;VZaAiA*-Yn$Y>Q(Ai`q1BP-VV6p;DP?()z5>yBjAa>G`c zzn>Z|Y$*evVGr2UJPk&z|4F@3BeU|)uISwDdJZLHR@$TB(fF$1%gRjbQ`7RJAHMv! z`+ME8BMCqUz{A66z;;RVZ(EI&&ev_;eL25Y*7T49cpgpGyA?d=zB>`2 zQd%ffR~J+-{P%-?9y?e`|IXNrdv>olD`SprSP0~O>Ei8rS^4DWcdSCh2IPL(5APlXA9W}Z6?c&njjXtb%6!*WnH7O%s&NeDmMI}vgSh|0>g4N6C z-?bknm8KnCHbgzUe;QA(e36_bu+fLFVwE#j$`O_%=OgLA2KS5aZBf5~d4(%*7bTwRrTTD|f;qB4db zDhg}m*^nA9#Rg5k>04^#9DlAmJ(*l}8Nbt`-AjFMq^jgsQv0TsuUoqF>Tv~~H}yw) zHQ!d;mDj^9Mh000b{(mT?&8FE#CHijRXKH>0zP3Z50=g)pzC0zT*Hd`WsGM_S7!uD z)-(LMn32QoTpjU7cHnwMaK_jG$wk81bg!UpeZ_xZa#4Tn=+f=;Z&-!nTT$^%=GY$ zB)PBTY{b4t5v8da^Q|g&6ma+gISL5yai_oZ_5b`@jLlB2b9+c;#>cXe;B)5Kl}Fg> zoFuA$cPV0=fXzhwOsmsE zcfh}<4}5(7p-_Z?N=EYP7ZiZ z%|geN!ok*Dt0^9@jnA7RqFcw)N__(YMcr*s*Uwmcy<`g<&1%}b=6>KBQnJqF^xbcQ z0xN!LSLd=`q|eG~I23Bx6aukAc2M!*3ZJ?qfm>`I`8)H;R(6W$aqW#dwo+L}9P5btx=_)Xu1$H#};GL&9je%y%MAPhyEzZ@8HM+)ghipvdk zhU*q(495TLQ7?$*{(P{}`J4q0p}%=T6pkeTo{ws zI|?;_h}%(@b8&SX6Q{9jCq0A*R@>^?Tsey%rLAXIrN8Qp3$OY#c&m^wDiNSUoT9yjxzi|IcgxHO`kfhUO{W(T`x%GWZsMEFdc}Q#Xq@2(XO~!B$1i8tkW!0DW7NLV z#wQ%*{u0;Pj1$(Q8p*=7w?rp6U4J$xR@*mZW;g&T406o+^^bx>858+I7TP7(EKzu; zDeH)XDz3?~4>!d;vhsvo)!2!$Hq$q{LmnQTHA+6Hz351~&o}+x>+N@jIX7ZY?`U;h z6aI{kqWzHIb|NFh0GsVan4skClJFPk2)#)D9X0LUP9nWY?WKB;mVTVWil4mpJKvvt zvYmr@3~HCz3N~F{Qy~4W?)H_tk?f}fDJ>(~9=6WwW!k5sr0X?K+;+saM7Gy72qP(` z9trqfrF~+inu_p%=lVjJfn&qn)ugWTYDEd(bL%x=1Lju{I=tpY2HxL6n67pDH!4Bk zuK*!d=`$6A0Bqvk7vF6dxXYu8%5RUI<@U;O;aIvN-bV7_6P$-dq9Qg1JG_#daB*zSu#j0&%4cTvR-@9su=5qf1xw&>dZO7aqJ?T~*{$WY@ zTV#hBDKb+&ScNcg0tW3rJ=<+|4br}dVUylwBi!J7Y|1TcX)5f2q_r<<8kz4suMR{U z3$Xj#_QSG+jTr1hdg)BvSE=`{kYl zOkZXyIhaAi7Fo+9hc#Dd!3$7Fufs8Zs*N9`09|@`ivM^3=(Ov1X&g%^}_LAL@9$E4t{>!K) zTbaz4mR+U|oED6oZYqq`G+eGSST;U7Kf4Nr>qLjcHw@7y>zZbalt_y5fgx|vXRlNW zs0~aXQs-2$#Tyir7|N!~Y-Akb(?`wN*90i4Lg7cFw@r?*YukIt`bjIwp{|CR=N|*! z(cU=a1JYz%*y$$+)&k6n3EAg4vy(;9c5TUp%K&7*#d=thfBvM8{tgy@f!{V-|}FknPmmq9ZiP7UnI%K1n-{N>)O-V*dlv$ zwEZdSY%tx+a$3@z9+^qhP((46p%>Amx=Qu~+wA6A=l~RETXW_$I>u+5V@i|@RoHh! zfEp>!Ah)H-sLiH!Boj=6UAB@%eMxM(amlP-E6e^Ug|(n%Ls9HXMhey+SDD!fOu;cI@eV%Xdpqhm*wP zg!lnxA~$i;YAG^uXfjkz@=Yl>x9q-j2)8(9a;fZ;XkEs5`Y*NUC4TvIYeWPPPLwaP zTvX(hF#U!n;NQ&fNHM!kEV!v6`;3h7!PGJ=$?KC+lYV<*0C~`gF-z%x{8=;5!}PIF zvUVFW%E}T$qpt4Y&N;L3>c2cMvpDOH)ihvAWnXad$=cL!JEVV(oOtnwZd+A);av~# zr)1y@cm~ed)Gmt!&L1s(o`dQ+f}d56Ry0+~BP2Ot+n2p*MjrK1ho{Sks7oRK$|k%u zrX)DU)J!bQmpoX5oaur$lvR_zsZjfHmG|V$OEFHgbcK2S?qRi=Q8dp_G*!cMXv1di rr^TkXmbtV^pup3pbB;3sc{7`WB@fJLV1XfiZDmy}X-?1=AQ1lth2(7X literal 0 HcmV?d00001 diff --git a/autotest/gcore/data/gtiff/sparse_nodata_one.tif b/autotest/gcore/data/gtiff/sparse_nodata_one.tif new file mode 100644 index 0000000000000000000000000000000000000000..cf7f3acd65dd56d4470ba71b0a3448fe5779efca GIT binary patch literal 158 zcmebD)MDUZU|`^3U|?isU<9(j7>UgUWrI{PBZ;#iu>~1f!1{nn7(|i8#gN$INNQw| U#DkI8jZ2sqn1E&&f(!xz0M$GLLI3~& literal 0 HcmV?d00001 diff --git a/autotest/gcore/data/gtiff/sparse_tiled_contig.tif b/autotest/gcore/data/gtiff/sparse_tiled_contig.tif new file mode 100644 index 0000000000000000000000000000000000000000..65558a65b933c10a9eba437f56cf8ba9141d2ff5 GIT binary patch literal 206 zcmebD)MDUZU|`^7U|?isU<9%>fS3`=76r1IfNW+Uy$Xn#q2eHQY*026kSz*jgVf0& zsd0j`6@cQ-P__V&?ZU{yzyhRq0I@4j9K_xL#KAx|$cz&}+_;2^0c5rz5Q6{*7(&>X KBqPx6j0^zm3kP}t literal 0 HcmV?d00001 diff --git a/autotest/gcore/data/gtiff/sparse_tiled_separate.tif b/autotest/gcore/data/gtiff/sparse_tiled_separate.tif new file mode 100644 index 0000000000000000000000000000000000000000..42672bd1a55deb60620401165ba42d79aa9a6f15 GIT binary patch literal 254 zcmebD)MDUZU|`^7U|?isU<9%>fS3`=76r1IfNW+Uy$Xn#q2eHQY*026kSz*jgVf0& zi912r3P5pZC|dx?c41^;-~rMnfY=o%4q|Tr;$R>fWX2C5Zd}5|05aPUh(Uk@3?XdF KNk*Vw85sa}eFw_` literal 0 HcmV?d00001 diff --git a/autotest/gcore/libertiff.py b/autotest/gcore/libertiff.py new file mode 100755 index 000000000000..f4346b47fb3f --- /dev/null +++ b/autotest/gcore/libertiff.py @@ -0,0 +1,739 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: Test LIBERTIFF driver +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + +import glob +import math +import os +import threading + +import gdaltest +import pytest + +from osgeo import gdal + +pytestmark = pytest.mark.require_driver("LIBERTIFF") + + +def libertiff_open(filename): + return gdal.OpenEx(filename, allowed_drivers=["LIBERTIFF"]) + + +def test_libertiff_basic(): + ds = libertiff_open("data/byte.tif") + assert ds.GetMetadataItem("AREA_OR_POINT") == "Area" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "26711" + assert ds.GetGeoTransform() == pytest.approx( + (440720.0, 60.0, 0.0, 3751320.0, 0.0, -60.0) + ) + assert ds.GetGCPCount() == 0 + assert ds.GetGCPSpatialRef() is None + assert len(ds.GetGCPs()) == 0 + assert ds.GetRasterBand(1).GetColorInterpretation() == gdal.GCI_GrayIndex + assert ds.GetRasterBand(1).GetColorTable() is None + assert ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_ALL_VALID + assert ds.GetRasterBand(1).GetMaskBand().ComputeRasterMinMax() == (255, 255) + assert ds.GetRasterBand(1).GetNoDataValue() is None + assert ds.GetRasterBand(1).GetOffset() is None + assert ds.GetRasterBand(1).GetScale() is None + assert ds.GetRasterBand(1).GetDescription() == "" + assert ds.GetRasterBand(1).GetUnitType() == "" + assert ds.GetRasterBand(1).GetMetadataItem("JPEGTABLES", "TIFF") is None + assert ds.GetRasterBand(1).GetMetadataItem("IFD_OFFSET", "TIFF") == "408" + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_OFFSET_0_0", "TIFF") == "8" + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_OFFSET_0_1", "TIFF") is None + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_OFFSET_1_0", "TIFF") is None + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_SIZE_0_0", "TIFF") == "400" + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_SIZE_0_1", "TIFF") is None + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_SIZE_1_0", "TIFF") is None + assert ( + ds.GetRasterBand(1).InterpolateAtPoint(0, 0, gdal.GRIORA_NearestNeighbour) + == 107.0 + ) + with pytest.raises(Exception): + ds.GetRasterBand(1).Fill(0) + + +@pytest.mark.require_driver("GTiff") +def test_libertiff_basic_compare_gtiff(): + ds = libertiff_open("data/byte.tif") + ds_ref = gdal.OpenEx("data/byte.tif", allowed_drivers=["GTiff"]) + assert ds.ReadRaster() == ds_ref.ReadRaster() + assert ds.ReadRaster(1, 2, 1, 1) == ds_ref.ReadRaster(1, 2, 1, 1) + assert ds.ReadRaster(band_list=[1, 1]) == ds_ref.ReadRaster(band_list=[1, 1]) + assert ds.ReadRaster(1, 2, 3, 4) == ds_ref.ReadRaster(1, 2, 3, 4) + assert ds.ReadRaster(buf_type=gdal.GDT_Int16) == ds_ref.ReadRaster( + buf_type=gdal.GDT_Int16 + ) + assert ds.ReadRaster(buf_xsize=3, buf_ysize=5) == ds_ref.ReadRaster( + buf_xsize=3, buf_ysize=5 + ) + assert ds.ReadRaster( + buf_xsize=3, buf_ysize=5, resample_alg=gdal.GRIORA_Bilinear + ) == ds_ref.ReadRaster(buf_xsize=3, buf_ysize=5, resample_alg=gdal.GRIORA_Bilinear) + + +@pytest.mark.require_driver("GTiff") +@pytest.mark.parametrize("GDAL_FORCE_CACHING", ["YES", "NO"]) +def test_libertiff_3band_separate(GDAL_FORCE_CACHING): + with gdal.config_option("GDAL_FORCE_CACHING", GDAL_FORCE_CACHING): + ds = libertiff_open("data/rgbsmall.tif") + ds_ref = gdal.OpenEx("data/rgbsmall.tif", allowed_drivers=["GTiff"]) + assert ds.ReadRaster() == ds_ref.ReadRaster() + assert ds.GetRasterBand(2).ReadRaster() == ds_ref.GetRasterBand(2).ReadRaster() + assert ds.ReadRaster(1, 2, 1, 1) == ds_ref.ReadRaster(1, 2, 1, 1) + assert ds.ReadRaster(band_list=[3, 2, 1]) == ds_ref.ReadRaster(band_list=[3, 2, 1]) + assert ds.ReadRaster(1, 2, 3, 4) == ds_ref.ReadRaster(1, 2, 3, 4) + assert ds.ReadRaster(buf_type=gdal.GDT_Int16) == ds_ref.ReadRaster( + buf_type=gdal.GDT_Int16 + ) + assert ds.ReadRaster(buf_xsize=3, buf_ysize=5) == ds_ref.ReadRaster( + buf_xsize=3, buf_ysize=5 + ) + assert ds.ReadRaster( + 1.5, 2.5, 6.5, 7.5, buf_xsize=3, buf_ysize=5, resample_alg=gdal.GRIORA_Bilinear + ) == ds_ref.ReadRaster( + 1.5, 2.5, 6.5, 7.5, buf_xsize=3, buf_ysize=5, resample_alg=gdal.GRIORA_Bilinear + ) + + +@pytest.mark.require_driver("GTiff") +@pytest.mark.parametrize("GDAL_FORCE_CACHING", ["YES", "NO"]) +def test_libertiff_4band_pixel_interleaved(GDAL_FORCE_CACHING): + with gdal.config_option("GDAL_FORCE_CACHING", GDAL_FORCE_CACHING): + ds = libertiff_open("data/stefan_full_rgba.tif") + ds_ref = gdal.OpenEx("data/stefan_full_rgba.tif", allowed_drivers=["GTiff"]) + assert ds.ReadRaster() == ds_ref.ReadRaster() + assert ds.ReadRaster(buf_pixel_space=4, buf_band_space=1) == ds_ref.ReadRaster( + buf_pixel_space=4, buf_band_space=1 + ) + assert ds.ReadRaster(band_list=[3, 2, 1, 4]) == ds_ref.ReadRaster( + band_list=[3, 2, 1, 4] + ) + assert ds.ReadRaster(band_list=[3, 2, 1]) == ds_ref.ReadRaster(band_list=[3, 2, 1]) + + +def test_libertiff_with_ovr(): + ds = libertiff_open("data/byte_with_ovr.tif") + assert ds.GetRasterBand(1).GetOverviewCount() == 2 + assert ds.GetRasterBand(1).GetOverview(-1) is None + assert ds.GetRasterBand(1).GetOverview(2) is None + assert ds.GetRasterBand(1).GetOverview(0).Checksum() == 1087 + assert ds.GetRasterBand(1).GetOverview(1).Checksum() == 328 + assert ( + ds.GetRasterBand(1).ReadRaster(buf_xsize=10, buf_ysize=10) + == ds.GetRasterBand(1).GetOverview(0).ReadRaster() + ) + + +@pytest.mark.require_driver("JPEG") +def test_libertiff_with_mask(): + ds = libertiff_open("data/ycbcr_with_mask.tif") + assert ds.GetSpatialRef() is not None + assert ds.RasterCount == 3 + assert ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET + assert ds.GetRasterBand(1).GetMaskBand().Checksum() == 1023 + assert ds.GetRasterBand(1).GetMetadataItem("JPEGTABLES", "TIFF") is not None + + +def test_libertiff_nodata(): + ds = libertiff_open("data/nan32_nodata.tif") + assert math.isnan(ds.GetRasterBand(1).GetNoDataValue()) + + +def test_libertiff_sparse_nodata_one(): + ds = libertiff_open("data/gtiff/sparse_nodata_one.tif") + assert ds.GetRasterBand(1).GetNoDataValue() == 1 + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_OFFSET_0_0", "TIFF") is None + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_SIZE_0_0", "TIFF") is None + assert ds.GetRasterBand(1).Checksum() == 1 + assert ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) == b"\x01" + assert ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) == b"\x01" + + +@pytest.mark.parametrize("filename", ["sparse_tiled_separate", "sparse_tiled_contig"]) +def test_libertiff_sparse_tiled(filename): + ds = libertiff_open(f"data/gtiff/{filename}.tif") + assert ds.GetRasterBand(1).ReadRaster() == b"\x01" * (40 * 21) + assert ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) == b"\x01" + + +def test_libertiff_metadata(): + ds = libertiff_open("data/complex_float32.tif") + assert ds.GetMetadata() == { + "bands": "1", + "byte_order": "0", + "data_type": "6", + "description": "File Imported into ENVI.", + "file_type": "ENVI Standard", + "header_offset": "0", + "interleave": "bsq", + "lines": "4148", + "samples": "9400", + "sensor_type": "Unknown", + "wavelength_units": "Unknown", + } + + +def test_libertiff_check_signed_int8(): + ds = libertiff_open("../gdrivers/data/gtiff/int8.tif") + assert ds.GetRasterBand(1).Checksum() == 1046 + + +@pytest.mark.parametrize( + "filename", ["int16.tif", "gtiff/int16_big_endian.tif", "int32.tif", "int64.tif"] +) +def test_libertiff_check_signed_int(filename): + ds = libertiff_open("data/" + filename) + assert ds.GetRasterBand(1).Checksum() == 4672 + + +@pytest.mark.parametrize( + "filename", + [ + "cint16.tif", + "cint32.tif", + "gtiff/cint32_big_endian.tif", + "cfloat32.tif", + "cfloat64.tif", + ], +) +def test_libertiff_check_complex(filename): + ds = libertiff_open("data/" + filename) + assert ds.GetRasterBand(1).Checksum() == 5028 + + +@pytest.mark.parametrize( + "filename_middle", + [ + "DEFLATE", + "DEFLATE_tiled", + "JPEG", + "JPEG_tiled", + "JXL", + "JXL_tiled", + "LERC_DEFLATE", + "LERC_DEFLATE_tiled", + "LERC", + "LERC_tiled", + "LERC_ZSTD", + "LERC_ZSTD_tiled", + "little_endian_golden", + "little_endian_tiled_lzw_golden", + "LZMA", + "LZMA_tiled", + "LZW_predictor_2", + "LZW", + "LZW_tiled", + "NONE", + "NONE_tiled", + "ZSTD", + "ZSTD_tiled", + ], +) +def test_libertiff_check_byte(filename_middle): + if "JPEG" in filename_middle and gdal.GetDriverByName("JPEG") is None: + pytest.skip("JPEG driver missing") + if "JXL" in filename_middle and gdal.GetDriverByName("JPEGXL") is None: + pytest.skip("JPEGXL driver missing") + if "WEBP" in filename_middle and gdal.GetDriverByName("WEBP") is None: + pytest.skip("WEBP driver missing") + if ( + "LZMA" in filename_middle + and gdal.GetDriverByName("LIBERTIFF").GetMetadataItem( + "LZMA_SUPPORT", "LIBERTIFF" + ) + is None + ): + pytest.skip("LZMA support missing") + if ( + "LERC" in filename_middle + and gdal.GetDriverByName("LIBERTIFF").GetMetadataItem( + "LERC_SUPPORT", "LIBERTIFF" + ) + is None + ): + pytest.skip("LERC support missing") + if ( + "ZSTD" in filename_middle + and gdal.GetDriverByName("LIBERTIFF").GetMetadataItem( + "ZSTD_SUPPORT", "LIBERTIFF" + ) + is None + ): + pytest.skip("ZSTD support missing") + + ds = libertiff_open(f"data/gtiff/byte_{filename_middle}.tif") + if "JPEG" in filename_middle: + assert ds.GetRasterBand(1).Checksum() > 0 + if gdal.GetDriverByName("GTiff"): + ds_ref = gdal.OpenEx( + f"data/gtiff/byte_{filename_middle}.tif", allowed_drivers=["GTiff"] + ) + assert ds.GetRasterBand(1).Checksum() == ds_ref.GetRasterBand(1).Checksum() + else: + assert ds.GetRasterBand(1).Checksum() == 4672 + + +@pytest.mark.parametrize( + "filename_middle", + [ + "byte_LZW_predictor_2", + "DEFLATE_separate", + "DEFLATE", + "DEFLATE_tiled_separate", + "DEFLATE_tiled", + "JPEG_separate", + "JPEG", + "JPEG_tiled_separate", + "JPEG_tiled", + "JPEG_ycbcr", + "JXL_separate", + "JXL", + "JXL_tiled_separate", + "JXL_tiled", + "LERC_DEFLATE_separate", + "LERC_DEFLATE", + "LERC_DEFLATE_tiled_separate", + "LERC_DEFLATE_tiled", + "LERC_separate", + "LERC", + "LERC_tiled_separate", + "LERC_tiled", + "LERC_ZSTD_separate", + "LERC_ZSTD", + "LERC_ZSTD_tiled_separate", + "LERC_ZSTD_tiled", + "LZMA_separate", + "LZMA", + "LZMA_tiled_separate", + "LZMA_tiled", + "LZW_separate", + "LZW", + "LZW_tiled_separate", + "LZW_tiled", + "NONE_separate", + "NONE", + "NONE_tiled_separate", + "NONE_tiled", + "uint16_LZW_predictor_2", + "uint32_LZW_predictor_2", + "uint64_LZW_predictor_2", + "WEBP", + "WEBP_tiled", + "ZSTD_separate", + "ZSTD", + "ZSTD_tiled_separate", + "ZSTD_tiled", + "int16_bigendian_lzw_predictor_2", + ], +) +def test_libertiff_check_rgbsmall(filename_middle): + if "JPEG" in filename_middle and gdal.GetDriverByName("JPEG") is None: + pytest.skip("JPEG driver missing") + if "JXL" in filename_middle and gdal.GetDriverByName("JPEGXL") is None: + pytest.skip("JPEGXL driver missing") + if "WEBP" in filename_middle and gdal.GetDriverByName("WEBP") is None: + pytest.skip("WEBP driver missing") + if ( + "LZMA" in filename_middle + and gdal.GetDriverByName("LIBERTIFF").GetMetadataItem( + "LZMA_SUPPORT", "LIBERTIFF" + ) + is None + ): + pytest.skip("LZMA support missing") + if ( + "LERC" in filename_middle + and gdal.GetDriverByName("LIBERTIFF").GetMetadataItem( + "LERC_SUPPORT", "LIBERTIFF" + ) + is None + ): + pytest.skip("LERC support missing") + if ( + "ZSTD" in filename_middle + and gdal.GetDriverByName("LIBERTIFF").GetMetadataItem( + "ZSTD_SUPPORT", "LIBERTIFF" + ) + is None + ): + pytest.skip("ZSTD support missing") + + ds = libertiff_open(f"data/gtiff/rgbsmall_{filename_middle}.tif") + if "JPEG" in filename_middle: + assert ds.GetRasterBand(1).Checksum() > 0 + if gdal.GetDriverByName("GTiff"): + ds_ref = gdal.OpenEx( + f"data/gtiff/rgbsmall_{filename_middle}.tif", + allowed_drivers=["GTiff"], + ) + assert [ds.GetRasterBand(i + 1).Checksum() for i in range(3)] == [ + ds_ref.GetRasterBand(i + 1).Checksum() for i in range(3) + ] + else: + assert [ds.GetRasterBand(i + 1).Checksum() for i in range(3)] == [ + 21212, + 21053, + 21349, + ] + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_OFFSET_0_0", "TIFF") is not None + assert ds.GetRasterBand(1).GetMetadataItem("BLOCK_SIZE_0_0", "TIFF") is not None + + +@pytest.mark.parametrize( + "filename_prefix", + [ + "byte_LZW_predictor_2", + "float32_LZW_predictor_2", + "float32_LZW_predictor_3", + "float32_lzw_predictor_3_big_endian", + "float64_LZW_predictor_2", + "float64_LZW_predictor_3", + "uint16_LZW_predictor_2", + "uint32_LZW_predictor_2", + "uint64_LZW_predictor_2", + ], +) +def test_libertiff_check_predictor_1_band(filename_prefix): + ds = libertiff_open(f"data/gtiff/{filename_prefix}.tif") + assert ds.GetRasterBand(1).Checksum() == 4672 + + +@pytest.mark.parametrize( + "filename_prefix", + [ + "stefan_full_greyalpha_byte_LZW_predictor_2", + "stefan_full_greyalpha_uint16_LZW_predictor_2", + "stefan_full_greyalpha_uint32_LZW_predictor_2", + "stefan_full_greyalpha_uint64_LZW_predictor_2", + ], +) +def test_libertiff_check_predictor_2_bands(filename_prefix): + ds = libertiff_open(f"data/gtiff/{filename_prefix}.tif") + assert [ds.GetRasterBand(i + 1).Checksum() for i in range(2)] == [1970, 10807] + + +@pytest.mark.parametrize( + "filename_prefix", + [ + "rgbsmall_byte_LZW_predictor_2", + "rgbsmall_uint16_LZW_predictor_2", + "rgbsmall_uint32_LZW_predictor_2", + "rgbsmall_uint64_LZW_predictor_2", + ], +) +def test_libertiff_check_predictor_3_bands(filename_prefix): + ds = libertiff_open(f"data/gtiff/{filename_prefix}.tif") + assert [ds.GetRasterBand(i + 1).Checksum() for i in range(3)] == [ + 21212, + 21053, + 21349, + ] + + +def test_libertiff_read_predictor_4_bands(): + ds = libertiff_open("data/gtiff/stefan_full_rgba_LZW_predictor_2.tif") + assert [ds.GetRasterBand(i + 1).Checksum() for i in range(4)] == [ + 12603, + 58561, + 36064, + 10807, + ] + + +def test_libertiff_read_predictor_5_bands(): + ds = libertiff_open("data/gtiff/byte_5_bands_LZW_predictor_2.tif") + assert [ds.GetRasterBand(i + 1).Checksum() for i in range(5)] == [ + 4672, + 4563, + 4672, + 4563, + 4672, + ] + + +############################################################################### +# Test reading a GeoTIFF file with tiepoints in PixelIsPoint format. + + +def test_libertiff_read_tiepoints_pixelispoint(): + + ds = libertiff_open("data/byte_gcp_pixelispoint.tif") + assert ds.GetMetadataItem("AREA_OR_POINT") == "Point" + assert ds.GetSpatialRef() is None + assert ds.GetGCPSpatialRef() is not None + assert ds.GetGCPCount() == 4 + gcp = ds.GetGCPs()[0] + assert ( + gcp.GCPPixel == pytest.approx(0.5, abs=1e-5) + and gcp.GCPLine == pytest.approx(0.5, abs=1e-5) + and gcp.GCPX == pytest.approx(-180, abs=1e-5) + and gcp.GCPY == pytest.approx(90, abs=1e-5) + and gcp.GCPZ == pytest.approx(0, abs=1e-5) + ) + + +############################################################################### +# Test reading a 4-band contig jpeg compressed geotiff. + + +@pytest.mark.require_driver("JPEG") +def test_libertiff_rgba_jpeg_contig(): + + ds = libertiff_open("data/stefan_full_rgba_jpeg_contig.tif") + assert ds.GetRasterBand(1).Checksum() > 0 + assert ds.GetRasterBand(2).Checksum() > 0 + assert ds.GetRasterBand(3).Checksum() > 0 + assert ds.GetRasterBand(4).Checksum() > 0 + + +############################################################################### +# Test reading a 12bit jpeg compressed geotiff. + + +@pytest.mark.skipif( + "SKIP_TIFF_JPEG12" in os.environ, reason="Crashes on build-windows-msys2-mingw" +) +@pytest.mark.require_driver("JPEG") +def test_libertiff_12bitjpeg(tmp_vsimem): + + jpeg_drv = gdal.GetDriverByName("JPEG") + if "UInt16" not in jpeg_drv.GetMetadataItem(gdal.DMD_CREATIONDATATYPES): + pytest.skip("12-bit JPEG not supported in this build") + + filename = str(tmp_vsimem / "tmp.tif") + gdal.FileFromMemBuffer( + filename, open("data/mandrilmini_12bitjpeg.tif", "rb").read() + ) + ds = libertiff_open(filename) + + stats = ds.GetRasterBand(1).ComputeStatistics(False) + assert not ( + stats[2] < 2150 or stats[2] > 2180 or str(stats[2]) == "nan" + ), "did not get expected mean for band1." + + +############################################################################### + + +@pytest.mark.parametrize( + "filename", ["data/excessive-memory-TIFFFillStrip.tif", "data/huge4GB.tif"] +) +def test_libertiff_read_error(filename): + + ds = libertiff_open(filename) + for i in range(ds.RasterCount): + with pytest.raises(Exception): + ds.GetRasterBand(i + 1).Checksum() + + +############################################################################### + + +@pytest.mark.parametrize( + "filename", + [ + "data/huge-number-strips.tiff", + ], +) +def test_libertiff_open_error(filename): + + with pytest.raises(Exception): + libertiff_open(filename) + + +############################################################################### +# Test we don't crash on files + + +@gdaltest.disable_exceptions() +def test_libertiff_test_all_tif(): + + for filename in glob.glob("data/*.tif"): + # print(filename) + with gdal.quiet_errors(): + ds = libertiff_open(filename) + if ds and ds.RasterCount: + ds.ReadRaster(0, 0, 1, 1) + + +def test_libertiff_thread_safe_readraster(): + + ds = libertiff_open("data/gtiff/rgbsmall_DEFLATE_tiled_separate.tif") + assert ds.IsThreadSafe(gdal.OF_RASTER) + + res = [True] + + def check(): + for i in range(100): + if [ds.GetRasterBand(i + 1).Checksum() for i in range(3)] != [ + 21212, + 21053, + 21349, + ]: + res[0] = False + + threads = [threading.Thread(target=check) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] + + +############################################################################### +# Test different datatypes for StripOffsets tag with little/big, classic/bigtiff + + +@pytest.mark.parametrize( + "filename,expected_offsets", + [ + ("data/classictiff_one_block_byte.tif", [158]), + ("data/classictiff_one_block_long.tif", [158]), + ("data/classictiff_one_block_be_long.tif", [158]), + ("data/classictiff_one_strip_long.tif", [146]), + ("data/classictiff_one_strip_be_long.tif", [146]), + ("data/classictiff_two_strip_short.tif", [162, 163]), + ("data/classictiff_two_strip_be_short.tif", [162, 163]), + ("data/classictiff_four_strip_short.tif", [178, 179, 180, 181]), + ("data/classictiff_four_strip_be_short.tif", [178, 179, 180, 181]), + ("data/bigtiff_four_strip_short.tif", [316, 317, 318, 319]), + ("data/bigtiff_four_strip_be_short.tif", [316, 317, 318, 319]), + ("data/bigtiff_one_block_long8.tif", [272]), + ("data/bigtiff_one_block_be_long8.tif", [272]), + ("data/bigtiff_one_strip_long.tif", [252]), + ("data/bigtiff_one_strip_be_long.tif", [252]), + ("data/bigtiff_one_strip_long8.tif", [252]), + ("data/bigtiff_one_strip_be_long8.tif", [252]), + ("data/bigtiff_two_strip_long.tif", [284, 285]), + ("data/bigtiff_two_strip_be_long.tif", [284, 285]), + ("data/bigtiff_two_strip_long8.tif", [284, 285]), + ("data/bigtiff_two_strip_be_long8.tif", [284, 285]), + ], +) +def test_libertiff_read_stripoffset_types(filename, expected_offsets): + + ds = libertiff_open(filename) + offsets = [] + for row in range(4): + mdi = ds.GetRasterBand(1).GetMetadataItem("BLOCK_OFFSET_0_%d" % row, "TIFF") + if mdi is None: + break + offsets.append(int(mdi)) + assert offsets == expected_offsets + assert ds.GetRasterBand(1).Checksum() > 0 + + +def test_libertiff_packbits(): + ds = libertiff_open("data/seperate_strip.tif") + assert [ds.GetRasterBand(i + 1).Checksum() for i in range(3)] == [ + 15234, + 15234, + 15234, + ] + + +def test_libertiff_coordinate_epoch(): + ds = libertiff_open("data/gtiff/byte_coord_epoch.tif") + assert ds.GetSpatialRef().GetCoordinateEpoch() == 2020.0 + + +@pytest.mark.require_driver("WEBP") +def test_libertiff_webp_lossy(): + ds = libertiff_open("data/tif_webp.tif") + assert ds.GetMetadata("IMAGE_STRUCTURE") == { + "COMPRESSION_REVERSIBILITY": "LOSSY", + "COMPRESSION": "WEBP", + "INTERLEAVE": "PIXEL", + } + + +def test_libertiff_multi_image(): + ds = libertiff_open("data/twoimages.tif") + assert ds.RasterCount == 0 + assert ds.GetSubDatasets() == [ + ("GTIFF_DIR:1:data/twoimages.tif", "Page 1 (20P x 20L x 1B)"), + ("GTIFF_DIR:2:data/twoimages.tif", "Page 2 (20P x 20L x 1B)"), + ] + + with pytest.raises(Exception): + libertiff_open("GTIFF_DIR:") + with pytest.raises(Exception): + libertiff_open("GTIFF_DIR:0") + with pytest.raises(Exception): + libertiff_open("GTIFF_DIR:-1:data/twoimages.tif") + with pytest.raises(Exception): + libertiff_open("GTIFF_DIR:0:data/twoimages.tif") + with pytest.raises(Exception): + libertiff_open("GTIFF_DIR:3:data/twoimages.tif") + with pytest.raises(Exception): + libertiff_open("GTIFF_DIR:1:i_do_not_exist.tif") + + ds = libertiff_open("GTIFF_DIR:1:data/twoimages.tif") + assert ds.GetRasterBand(1).GetMetadataItem("IFD_OFFSET", "TIFF") == "408" + + ds = libertiff_open("GTIFF_DIR:2:data/twoimages.tif") + assert ds.GetRasterBand(1).GetMetadataItem("IFD_OFFSET", "TIFF") == "958" + + +def test_libertiff_miniswhite(): + ds = libertiff_open("data/gtiff/miniswhite.tif") + assert ds.GetMetadataItem("MINISWHITE", "IMAGE_STRUCTURE") == "YES" + assert ds.GetRasterBand(1).GetColorInterpretation() == gdal.GCI_PaletteIndex + ct = ds.GetRasterBand(1).GetColorTable() + assert ct + assert ct.GetCount() == 2 + assert ct.GetColorEntry(0) == (255, 255, 255, 255) + assert ct.GetColorEntry(1) == (0, 0, 0, 255) + + +def test_libertiff_colortable(): + ds = libertiff_open("data/test_average_palette.tif") + assert ds.GetRasterBand(1).GetColorInterpretation() == gdal.GCI_PaletteIndex + ct = ds.GetRasterBand(1).GetColorTable() + assert ct + assert ct.GetCount() == 256 + assert ct.GetColorEntry(0) == (0, 0, 0, 255) + assert ct.GetColorEntry(1) == (255, 255, 255, 255) + assert ct.GetColorEntry(2) == (127, 127, 127, 255) + + +############################################################################### +# Corrupt all bytes of an existing file + + +def test_libertiff_corrupted(tmp_vsimem): + + filename = str(tmp_vsimem / "out.tif") + data = open("data/gtiff/miniswhite.tif", "rb").read() + gdal.FileFromMemBuffer(filename, data) + f = gdal.VSIFOpenL(filename, "rb+") + try: + for i in range(len(data)): + gdal.VSIFSeekL(f, i, 0) + ori_val = gdal.VSIFReadL(1, 1, f) + + for new_val in (b"\x00", b"\x01", b"\x7F", b"\xFE", b"\xFF"): + gdal.VSIFSeekL(f, i, 0) + gdal.VSIFWriteL(new_val, 1, 1, f) + try: + ds = libertiff_open(filename) + ds.ReadRaster() + except Exception: + pass + + # Restore old value + gdal.VSIFSeekL(f, i, 0) + gdal.VSIFWriteL(ori_val, 1, 1, f) + finally: + gdal.VSIFCloseL(f) diff --git a/autotest/pymod/gdaltest.py b/autotest/pymod/gdaltest.py index 77da5d307872..a17d6bce3ada 100755 --- a/autotest/pymod/gdaltest.py +++ b/autotest/pymod/gdaltest.py @@ -218,6 +218,14 @@ def testOpen( drv_name.lower() == "biggif" and self.drivername.lower() == "gif" ) + or ( + drv_name.lower() == "gtiff" + and self.drivername.lower() == "libertiff" + ) + or ( + drv_name.lower() == "libertiff" + and self.drivername.lower() == "gtiff" + ) ): drivers += [drv_name] other_ds = None diff --git a/doc/source/drivers/raster/gtiff.rst b/doc/source/drivers/raster/gtiff.rst index f6d4f722c42a..5f685fec05e6 100644 --- a/doc/source/drivers/raster/gtiff.rst +++ b/doc/source/drivers/raster/gtiff.rst @@ -25,6 +25,9 @@ file, such as YCbCr color model files, are automatically translated into RGBA (red, green, blue, alpha) form, and treated as four eight bit bands. +For an alternative which offers thread-safe read-only capabilities, consult +:ref:`raster.libertiff`. + Driver capabilities ------------------- diff --git a/doc/source/drivers/raster/index.rst b/doc/source/drivers/raster/index.rst index 0e511f99951f..7231e3208a3e 100644 --- a/doc/source/drivers/raster/index.rst +++ b/doc/source/drivers/raster/index.rst @@ -109,6 +109,7 @@ Raster drivers l1b lcp leveller + libertiff loslas map marfa diff --git a/doc/source/drivers/raster/libertiff.rst b/doc/source/drivers/raster/libertiff.rst new file mode 100644 index 000000000000..9bbf62469f1a --- /dev/null +++ b/doc/source/drivers/raster/libertiff.rst @@ -0,0 +1,36 @@ +.. _raster.libertiff: + +================================================================================ +LIBERTIFF -- GeoTIFF File Format +================================================================================ + +.. versionadded:: 3.11 + +.. shortname:: LIBERTIFF + +.. built_in_by_default:: + +This driver is an alternative to the default :ref:`raster.gtiff` driver, which +is natively thread-safe. Note that the driver is read-only. + +The driver is registered after the GTiff one. Consequently one must explicitly +specify ``LIBERTIFF`` in the allowed drivers of the :cpp:func:`GDALOpen`, or +with the ``-if`` option of command line utilities, to use it. + +The driver supports the following compression methods: LZW, Deflate, PackBits, +LZMA, ZSTD, LERC, JPEG, WEBP and JPEGXL. +The driver supports only BitsPerSample values of 1, 8, 16, 32, 64. + +The driver does *not* read any side-car file: ``.aux.xml``, ``.ovr``, ``.msk``, +``.imd``, etc. + +The driver mostly by-passes the GDAL raster block cache, and only caches (per-thread) +the last tile or strip it has read. Read patterns must be adapted accordingly, +to avoid repeated data acquisition from storage and decompression. + +Driver capabilities +------------------- + +.. supports_georeferencing:: + +.. supports_virtualio:: diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index 23a5a3a78607..a484587b6d55 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -79,6 +79,7 @@ if (CMAKE_BUILD_TYPE MATCHES "Debug" OR GDAL_ENABLE_DRIVER_NULL) gdal_optional_format(null "NULL dummy driver") endif () +gdal_optional_format(libertiff "GeoTIFF support through libertiff library") gdal_optional_format(hfa "Erdas Imagine .img") gdal_optional_format(sdts "SDTS translator") gdal_optional_format(nitf "National Imagery Transmission Format") diff --git a/frmts/drivers.ini b/frmts/drivers.ini index 11d381945755..c28e48407382 100644 --- a/frmts/drivers.ini +++ b/frmts/drivers.ini @@ -17,6 +17,7 @@ GTI SNAP_TIFF GTiff COG +LIBERTIFF NITF RPFTOC ECRGTOC diff --git a/frmts/gdalallregister.cpp b/frmts/gdalallregister.cpp index 005fd734ce89..a8f073396e17 100644 --- a/frmts/gdalallregister.cpp +++ b/frmts/gdalallregister.cpp @@ -332,6 +332,10 @@ void CPL_STDCALL GDALAllRegister() GDALRegister_COG(); #endif +#ifdef FRMT_libertiff + GDALRegister_LIBERTIFF(); +#endif + #ifdef FRMT_nitf GDALRegister_NITF(); GDALRegister_RPFTOC(); diff --git a/frmts/gtiff/libtiff/tif_lerc.c b/frmts/gtiff/libtiff/tif_lerc.c index d57b1d253de5..de824c7092b8 100644 --- a/frmts/gtiff/libtiff/tif_lerc.c +++ b/frmts/gtiff/libtiff/tif_lerc.c @@ -93,7 +93,6 @@ typedef struct #define LERCDecoderState(tif) GetLERCState(tif) #define LERCEncoderState(tif) GetLERCState(tif) -static int LERCEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s); static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s); static int LERCFixupTags(TIFF *tif) @@ -222,7 +221,7 @@ static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) { TIFFErrorExtR(tif, module, "Too large uncompressed strip/tile"); _TIFFfreeExt(tif, sp->uncompressed_buffer); - sp->uncompressed_buffer = 0; + sp->uncompressed_buffer = NULL; sp->uncompressed_alloc = 0; return 0; } @@ -230,12 +229,12 @@ static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) if (sp->uncompressed_alloc < new_alloc) { _TIFFfreeExt(tif, sp->uncompressed_buffer); - sp->uncompressed_buffer = _TIFFmallocExt(tif, new_alloc); + sp->uncompressed_buffer = (uint8_t *)_TIFFmallocExt(tif, new_alloc); if (!sp->uncompressed_buffer) { TIFFErrorExtR(tif, module, "Cannot allocate buffer"); _TIFFfreeExt(tif, sp->uncompressed_buffer); - sp->uncompressed_buffer = 0; + sp->uncompressed_buffer = NULL; sp->uncompressed_alloc = 0; return 0; } @@ -267,7 +266,7 @@ static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) TIFFErrorExtR(tif, module, "Cannot allocate buffer"); sp->mask_size = 0; _TIFFfreeExt(tif, sp->uncompressed_buffer); - sp->uncompressed_buffer = 0; + sp->uncompressed_buffer = NULL; sp->uncompressed_alloc = 0; return 0; } @@ -348,7 +347,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) return 0; } assert(lerc_data_sizet == (unsigned int)lerc_data_sizet); - lerc_data = sp->compressed_buffer; + lerc_data = (uint8_t *)sp->compressed_buffer; lerc_data_size = (unsigned int)lerc_data_sizet; #else z_stream strm; @@ -369,7 +368,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) strm.avail_in = (uInt)tif->tif_rawcc; strm.next_in = tif->tif_rawcp; strm.avail_out = sp->compressed_size; - strm.next_out = sp->compressed_buffer; + strm.next_out = (Bytef *)sp->compressed_buffer; zlib_ret = inflate(&strm, Z_FINISH); if (zlib_ret != Z_STREAM_END && zlib_ret != Z_OK) { @@ -377,7 +376,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) inflateEnd(&strm); return 0; } - lerc_data = sp->compressed_buffer; + lerc_data = (uint8_t *)sp->compressed_buffer; lerc_data_size = sp->compressed_size - strm.avail_out; inflateEnd(&strm); #endif @@ -396,7 +395,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) return 0; } - lerc_data = sp->compressed_buffer; + lerc_data = (uint8_t *)sp->compressed_buffer; lerc_data_size = (unsigned int)zstd_ret; #else TIFFErrorExtR(tif, module, "ZSTD support missing"); @@ -568,7 +567,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) { _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); sp->uncompressed_buffer_multiband = - _TIFFmallocExt(tif, num_bytes_needed); + (uint8_t *)_TIFFmallocExt(tif, num_bytes_needed); if (!sp->uncompressed_buffer_multiband) { sp->uncompressed_buffer_multiband_alloc = 0; @@ -752,7 +751,7 @@ static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) assert(sp != NULL); assert(sp->state == LSTATE_INIT_DECODE); - if (sp->uncompressed_buffer == 0) + if (sp->uncompressed_buffer == NULL) { memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Uncompressed buffer not allocated"); @@ -773,6 +772,8 @@ static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) return 1; } +#ifndef LERC_READ_ONLY + static int LERCSetupEncode(TIFF *tif) { LERCState *sp = LERCEncoderState(tif); @@ -1005,7 +1006,7 @@ static int LERCPostEncode(TIFF *tif) { _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); sp->uncompressed_buffer_multiband = - _TIFFmallocExt(tif, num_bytes_needed); + (uint8_t *)_TIFFmallocExt(tif, num_bytes_needed); if (!sp->uncompressed_buffer_multiband) { sp->uncompressed_buffer_multiband_alloc = 0; @@ -1126,7 +1127,8 @@ static int LERCPostEncode(TIFF *tif) sp->uncompressed_buffer_multiband, sp->lerc_version, GetLercDataType(tif), 1, sp->segment_width, sp->segment_height, dst_nbands, dst_nbands, sp->mask_buffer, sp->maxzerror, - sp->compressed_buffer, sp->compressed_size, &numBytesWritten); + (unsigned char *)sp->compressed_buffer, sp->compressed_size, + &numBytesWritten); } else #endif @@ -1139,7 +1141,8 @@ static int LERCPostEncode(TIFF *tif) use_mask ? 1 : 0, #endif use_mask ? sp->mask_buffer : NULL, sp->maxzerror, - sp->compressed_buffer, sp->compressed_size, &numBytesWritten); + (unsigned char *)sp->compressed_buffer, sp->compressed_size, + &numBytesWritten); } if (lerc_ret != 0) { @@ -1271,7 +1274,7 @@ static int LERCPostEncode(TIFF *tif) { int ret; uint8_t *tif_rawdata_backup = tif->tif_rawdata; - tif->tif_rawdata = sp->compressed_buffer; + tif->tif_rawdata = (uint8_t *)sp->compressed_buffer; tif->tif_rawcc = numBytesWritten; ret = TIFFFlushData1(tif); tif->tif_rawdata = tif_rawdata_backup; @@ -1282,11 +1285,13 @@ static int LERCPostEncode(TIFF *tif) return 1; } +#endif /* LERC_READ_ONLY */ + static void LERCCleanup(TIFF *tif) { LERCState *sp = GetLERCState(tif); - assert(sp != 0); + assert(sp != NULL); tif->tif_tagmethods.vgetfield = sp->vgetparent; tif->tif_tagmethods.vsetfield = sp->vsetparent; @@ -1312,20 +1317,21 @@ static void LERCCleanup(TIFF *tif) static const TIFFField LERCFields[] = { {TIFFTAG_LERC_PARAMETERS, TIFF_VARIABLE2, TIFF_VARIABLE2, TIFF_LONG, 0, TIFF_SETGET_C32_UINT32, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, FALSE, TRUE, - "LercParameters", NULL}, + (char *)"LercParameters", NULL}, {TIFFTAG_LERC_MAXZERROR, 0, 0, TIFF_ANY, 0, TIFF_SETGET_DOUBLE, - TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, "LercMaximumError", - NULL}, + TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, + (char *)"LercMaximumError", NULL}, {TIFFTAG_LERC_VERSION, 0, 0, TIFF_ANY, 0, TIFF_SETGET_UINT32, - TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, FALSE, FALSE, "LercVersion", NULL}, + TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, FALSE, FALSE, (char *)"LercVersion", + NULL}, {TIFFTAG_LERC_ADD_COMPRESSION, 0, 0, TIFF_ANY, 0, TIFF_SETGET_UINT32, TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, FALSE, FALSE, - "LercAdditionalCompression", NULL}, + (char *)"LercAdditionalCompression", NULL}, {TIFFTAG_ZSTD_LEVEL, 0, 0, TIFF_ANY, 0, TIFF_SETGET_INT, TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, - "ZSTD zstd_compress_level", NULL}, + (char *)"ZSTD zstd_compress_level", NULL}, {TIFFTAG_ZIPQUALITY, 0, 0, TIFF_ANY, 0, TIFF_SETGET_INT, - TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, "", NULL}, + TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, (char *)"", NULL}, }; static int LERCVSetFieldBase(TIFF *tif, uint32_t tag, ...) @@ -1517,12 +1523,14 @@ int TIFFInitLERC(TIFF *tif, int scheme) tif->tif_decoderow = LERCDecode; tif->tif_decodestrip = LERCDecode; tif->tif_decodetile = LERCDecode; +#ifndef LERC_READ_ONLY tif->tif_setupencode = LERCSetupEncode; tif->tif_preencode = LERCPreEncode; tif->tif_postencode = LERCPostEncode; tif->tif_encoderow = LERCEncode; tif->tif_encodestrip = LERCEncode; tif->tif_encodetile = LERCEncode; +#endif tif->tif_cleanup = LERCCleanup; /* Default values for codec-specific fields */ diff --git a/frmts/gtiff/libtiff/tif_lzw.c b/frmts/gtiff/libtiff/tif_lzw.c index 4baf78e50b20..c15c7eebc9e4 100644 --- a/frmts/gtiff/libtiff/tif_lzw.c +++ b/frmts/gtiff/libtiff/tif_lzw.c @@ -168,7 +168,6 @@ static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s); #ifdef LZW_COMPAT static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s); #endif -static void cl_hash(LZWCodecState *); /* * LZW Decoder. @@ -1017,6 +1016,10 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) } #endif /* LZW_COMPAT */ +#ifndef LZW_READ_ONLY + +static void cl_hash(LZWCodecState *); + /* * LZW Encoding. */ @@ -1373,11 +1376,13 @@ static void cl_hash(LZWCodecState *sp) hp->hash = -1; } +#endif + static void LZWCleanup(TIFF *tif) { (void)TIFFPredictorCleanup(tif); - assert(tif->tif_data != 0); + assert(tif->tif_data != NULL); if (LZWDecoderState(tif)->dec_codetab) _TIFFfreeExt(tif, LZWDecoderState(tif)->dec_codetab); @@ -1416,12 +1421,14 @@ int TIFFInitLZW(TIFF *tif, int scheme) tif->tif_decoderow = LZWDecode; tif->tif_decodestrip = LZWDecode; tif->tif_decodetile = LZWDecode; +#ifndef LZW_READ_ONLY tif->tif_setupencode = LZWSetupEncode; tif->tif_preencode = LZWPreEncode; tif->tif_postencode = LZWPostEncode; tif->tif_encoderow = LZWEncode; tif->tif_encodestrip = LZWEncode; tif->tif_encodetile = LZWEncode; +#endif tif->tif_cleanup = LZWCleanup; /* * Setup predictor setup. diff --git a/frmts/gtiff/libtiff/tif_packbits.c b/frmts/gtiff/libtiff/tif_packbits.c index 1ae50cbd47ce..d7db9b64ea36 100644 --- a/frmts/gtiff/libtiff/tif_packbits.c +++ b/frmts/gtiff/libtiff/tif_packbits.c @@ -31,6 +31,8 @@ */ #include +#ifndef PACKBITS_READ_ONLY + static int PackBitsPreEncode(TIFF *tif, uint16_t s) { (void)s; @@ -78,7 +80,7 @@ static int PackBitsEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) op = tif->tif_rawcp; ep = tif->tif_rawdata + tif->tif_rawdatasize; state = BASE; - lastliteral = 0; + lastliteral = NULL; while (cc > 0) { /* @@ -231,6 +233,8 @@ static int PackBitsEncodeChunk(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return (1); } +#endif + static int PackBitsDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { static const char module[] = "PackBitsDecode"; @@ -314,11 +318,13 @@ int TIFFInitPackBits(TIFF *tif, int scheme) tif->tif_decoderow = PackBitsDecode; tif->tif_decodestrip = PackBitsDecode; tif->tif_decodetile = PackBitsDecode; +#ifndef PACKBITS_READ_ONLY tif->tif_preencode = PackBitsPreEncode; tif->tif_postencode = PackBitsPostEncode; tif->tif_encoderow = PackBitsEncode; tif->tif_encodestrip = PackBitsEncodeChunk; tif->tif_encodetile = PackBitsEncodeChunk; +#endif return (1); } #endif /* PACKBITS_SUPPORT */ diff --git a/frmts/libertiff/CMakeLists.txt b/frmts/libertiff/CMakeLists.txt new file mode 100644 index 000000000000..0d24507c4ee8 --- /dev/null +++ b/frmts/libertiff/CMakeLists.txt @@ -0,0 +1,46 @@ +add_gdal_driver( + TARGET gdal_LIBERTIFF + SOURCES libertiffdataset.cpp + STRONG_CXX_WFLAGS + PLUGIN_CAPABLE_IF + "NOT GDAL_USE_LERC_INTERNAL\\\;NOT GDAL_USE_ZLIB_INTERNAL" + NO_DEPS) + +gdal_standard_includes(gdal_LIBERTIFF) +target_include_directories(gdal_LIBERTIFF PRIVATE + ${GDAL_RASTER_FORMAT_SOURCE_DIR}/mem + ${GDAL_RASTER_FORMAT_SOURCE_DIR}/gtiff/libtiff + ${PROJECT_SOURCE_DIR}/third_party/libertiff +) + +# Include first internal libraries +if (GDAL_USE_ZLIB_INTERNAL) + gdal_add_vendored_lib(gdal_LIBERTIFF libz) +endif () + +if (GDAL_USE_LERC_INTERNAL) + target_compile_definitions(gdal_LIBERTIFF PRIVATE -DLERC_SUPPORT) + gdal_add_vendored_lib(gdal_LIBERTIFF lerc) +endif () + +# Now external libraries + +if (NOT GDAL_USE_ZLIB_INTERNAL) + gdal_target_link_libraries(gdal_LIBERTIFF PRIVATE ZLIB::ZLIB) +endif () + +if (GDAL_USE_DEFLATE) + target_compile_definitions(gdal_LIBERTIFF PRIVATE -DLIBDEFLATE_SUPPORT) + gdal_target_link_libraries(gdal_LIBERTIFF PRIVATE Deflate::Deflate) +endif () + +if (NOT GDAL_USE_LERC_INTERNAL AND GDAL_USE_LERC) + target_compile_definitions(gdal_LIBERTIFF PRIVATE -DLERC_SUPPORT) + gdal_target_link_libraries(gdal_LIBERTIFF PRIVATE LERC::LERC) +endif () + +if (GDAL_USE_ZSTD) + target_compile_definitions(gdal_LIBERTIFF PRIVATE -DZSTD_SUPPORT) + gdal_target_link_libraries(gdal_LIBERTIFF PRIVATE ${ZSTD_TARGET}) +endif () + diff --git a/frmts/libertiff/libertiffdataset.cpp b/frmts/libertiff/libertiffdataset.cpp new file mode 100644 index 000000000000..776115387bf0 --- /dev/null +++ b/frmts/libertiff/libertiffdataset.cpp @@ -0,0 +1,3258 @@ +/****************************************************************************** + * + * Project: GDAL Core + * Purpose: GeoTIFF thread safe reader using libertiff library. + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "cpl_compressor.h" +#include "cpl_mem_cache.h" +#include "cpl_minixml.h" +#include "cpl_vsi_virtual.h" + +#include +#include +#include +#include +#include +#include + +#include "gdal_pam.h" +#include "gdal_interpolateatpoint.h" +#include "memdataset.h" + +#define LIBERTIFF_NS GDALLibertiffDataset +#include "libertiff.hpp" + +#include "libtiff_codecs.h" + +#define STRINGIFY(x) #x +#define XSTRINGIFY(x) STRINGIFY(x) + +/************************************************************************/ +/* LIBERTIFFDatasetFileReader */ +/************************************************************************/ + +struct LIBERTIFFDatasetFileReader final : public LIBERTIFF_NS::FileReader +{ + VSILFILE *const m_fp; + const bool m_bHasPread; + mutable bool m_bPReadAllowed = false; + mutable uint64_t m_nFileSize = 0; + mutable std::mutex m_oMutex{}; + + explicit LIBERTIFFDatasetFileReader(VSILFILE *fp) + : m_fp(fp), m_bHasPread(m_fp->HasPRead()) + { + } + + uint64_t size() const override + { + if (m_nFileSize == 0) + { + std::lock_guard oLock(m_oMutex); + // cppcheck-suppress identicalInnerCondition + if (m_nFileSize == 0) + { + m_fp->Seek(0, SEEK_END); + m_nFileSize = m_fp->Tell(); + } + } + return m_nFileSize; + } + + size_t read(uint64_t offset, size_t count, void *buffer) const override + { + if (m_bHasPread && m_bPReadAllowed) + { + return m_fp->PRead(buffer, count, offset); + } + else + { + std::lock_guard oLock(m_oMutex); + return m_fp->Seek(offset, SEEK_SET) == 0 + ? m_fp->Read(buffer, 1, count) + : 0; + } + } + + void setPReadAllowed() const + { + m_bPReadAllowed = true; + } + + CPL_DISALLOW_COPY_ASSIGN(LIBERTIFFDatasetFileReader) +}; + +/************************************************************************/ +/* LIBERTIFFDataset */ +/************************************************************************/ + +class LIBERTIFFDataset final : public GDALPamDataset +{ + public: + LIBERTIFFDataset() = default; + + static int Identify(GDALOpenInfo *poOpenInfo); + static GDALDataset *OpenStatic(GDALOpenInfo *poOpenInfo); + + const OGRSpatialReference *GetSpatialRef() const override + { + return m_aoGCPs.empty() && !m_oSRS.IsEmpty() ? &m_oSRS : nullptr; + } + + CPLErr GetGeoTransform(double *padfGeoTransform) override + { + memcpy(padfGeoTransform, m_geotransform.data(), + m_geotransform.size() * sizeof(double)); + return m_geotransformValid ? CE_None : CE_Failure; + } + + int GetGCPCount() override + { + return static_cast(m_aoGCPs.size()); + } + + const OGRSpatialReference *GetGCPSpatialRef() const override + { + return !m_aoGCPs.empty() && !m_oSRS.IsEmpty() ? &m_oSRS : nullptr; + } + + const GDAL_GCP *GetGCPs() override + { + return gdal::GCP::c_ptr(m_aoGCPs); + } + + protected: + CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + BANDMAP_TYPE panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg) override; + + CPLErr BlockBasedRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, + int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + const int *panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg) override + { + return IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, + nBufXSize, nBufYSize, eBufType, nBandCount, + const_cast(panBandMap), nPixelSpace, + nLineSpace, nBandSpace, psExtraArg); + } + + private: + friend class LIBERTIFFBand; + VSIVirtualHandleUniquePtr m_poFile{}; + std::shared_ptr m_fileReader{}; + std::unique_ptr m_image{}; + const CPLCompressor *m_decompressor = nullptr; + std::shared_ptr m_validityPtr = std::make_shared(0); + OGRSpatialReference m_oSRS{}; + bool m_geotransformValid = false; + std::array m_geotransform{1, 0, 0, 0, 0, 1}; + std::vector m_aoGCPs{}; + std::vector> m_apoOvrDSOwned{}; + std::vector m_apoOvrDS{}; + GDALRasterBand *m_poAlphaBand = nullptr; + std::unique_ptr m_poMaskDS{}; + bool m_bExpand1To255 = false; + std::vector m_jpegTablesOri{}; + std::vector m_jpegTables{}; + std::vector m_tileOffsets{}; + std::vector m_tileOffsets64{}; + std::vector m_tileByteCounts{}; + int m_lercVersion = LERC_VERSION_2_4; + int m_lercAdditionalCompression = LERC_ADD_COMPRESSION_NONE; + std::vector m_extraSamples{}; + + struct ThreadLocalState + { + private: + std::weak_ptr m_validityTest{}; + + public: + explicit ThreadLocalState(const LIBERTIFFDataset *ds) + : m_validityTest(ds->m_validityPtr) + { + memset(&m_tiff, 0, sizeof(m_tiff)); + } + + ~ThreadLocalState() + { + if (m_tiff.tif_cleanup) + m_tiff.tif_cleanup(&m_tiff); + } + + inline bool isValid() const + { + return m_validityTest.lock() != nullptr; + } + + // Used by IRasterIO() + std::vector m_abyIRasterIOBuffer{}; + + // Used by ReadBlock() + uint64_t m_curStrileIdx = std::numeric_limits::max(); + bool m_curStrileMissing = false; + std::vector m_decompressedBuffer{}; + std::vector m_compressedBuffer{}; + std::vector m_bufferForOneBitExpansion{}; + std::vector m_apabyDest{}; + std::vector m_floatingPointHorizPredictorDecodeTmpBuffer{}; + + TIFF m_tiff{}; + }; + + GDALDataType ComputeGDALDataType() const; + bool ProcessCompressionMethod(); + + bool Open(std::unique_ptr image); + + bool Open(GDALOpenInfo *poOpenInfo); + + ThreadLocalState &GetTLSState() const; + + void ReadSRS(); + void ReadGeoTransform(); + + bool ReadBlock(GByte *pabyBlockData, int nBlockXOff, int nBlockYOff, + int nBandCount, BANDMAP_TYPE panBandMap, + GDALDataType eBufType, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace) const; + + CPL_DISALLOW_COPY_ASSIGN(LIBERTIFFDataset) +}; + +/************************************************************************/ +/* LIBERTIFFBand */ +/************************************************************************/ + +class LIBERTIFFBand final : public GDALPamRasterBand +{ + public: + LIBERTIFFBand(LIBERTIFFDataset *poDSIn, int nBandIn, GDALDataType eDT, + int nBlockXSizeIn, int nBlockYSizeIn) + { + poDS = poDSIn; + nBand = nBandIn; + eDataType = eDT; + nBlockXSize = nBlockXSizeIn; + nBlockYSize = nBlockYSizeIn; + } + + double GetNoDataValue(int *pbHasNoData) override + { + if (pbHasNoData) + *pbHasNoData = m_bHasNoData; + return m_dfNoData; + } + + double GetScale(int *pbHasNoData) override + { + if (pbHasNoData) + *pbHasNoData = m_bHaveOffsetScale; + return m_dfScale; + } + + double GetOffset(int *pbHasNoData) override + { + if (pbHasNoData) + *pbHasNoData = m_bHaveOffsetScale; + return m_dfOffset; + } + + const char *GetDescription() const override + { + return m_osDescription.c_str(); + } + + const char *GetUnitType() override + { + return m_osUnitType.c_str(); + } + + GDALColorInterp GetColorInterpretation() override + { + return m_eColorInterp; + } + + GDALColorTable *GetColorTable() override + { + return m_poColorTable.get(); + } + + int GetOverviewCount() override + { + auto l_poDS = cpl::down_cast(poDS); + return static_cast(l_poDS->m_apoOvrDS.size()); + } + + GDALRasterBand *GetOverview(int idx) override + { + auto l_poDS = cpl::down_cast(poDS); + if (idx >= 0 && idx < GetOverviewCount()) + return l_poDS->m_apoOvrDS[idx]->GetRasterBand(nBand); + return nullptr; + } + + int GetMaskFlags() override + { + return nMaskFlags; + } + + GDALRasterBand *GetMaskBand() override + { + return poMask.get(); + } + + const char *GetMetadataItem(const char *pszName, + const char *pszDomain) override; + + CPLErr InterpolateAtPoint(double dfPixel, double dfLine, + GDALRIOResampleAlg eInterpolation, + double *pdfRealValue, + double *pdfImagValue = nullptr) const override; + + // We could do a smarter implementation by manually managing blocks in + // the TLS structure, but given we should rarely use that method, the + // current approach with a mutex should be good enouh + GDALRasterBlock *GetLockedBlockRef(int nXBlockOff, int nYBlockOff, + int bJustInitialize = FALSE) override + { + if (!m_bDebugGetLockedBlockRef) + { + m_bDebugGetLockedBlockRef = true; + CPLDebug("LIBERTIFF", "GetLockedBlockRef() called"); + } + std::lock_guard oLock(m_oMutexBlockCache); + return GDALRasterBand::GetLockedBlockRef(nXBlockOff, nYBlockOff, + bJustInitialize); + } + + GDALRasterBlock *TryGetLockedBlockRef(int nXBlockOff, + int nYBlockOff) override + { + std::lock_guard oLock(m_oMutexBlockCache); + return GDALRasterBand::TryGetLockedBlockRef(nXBlockOff, nYBlockOff); + } + + CPLErr FlushBlock(int nXBlockOff, int nYBlockOff, + int bWriteDirtyBlock = TRUE) override + { + std::lock_guard oLock(m_oMutexBlockCache); + return GDALRasterBand::FlushBlock(nXBlockOff, nYBlockOff, + bWriteDirtyBlock); + } + + protected: + CPLErr IReadBlock(int nBlockXOff, int nBlockYOff, void *pData) override; + + CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, GSpacing nPixelSpace, + GSpacing nLineSpace, + GDALRasterIOExtraArg *psExtraArg) override + { + int anBand[] = {nBand}; + return cpl::down_cast(poDS)->IRasterIO( + eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, nBufYSize, + eBufType, 1, anBand, nPixelSpace, nLineSpace, 0, psExtraArg); + } + + private: + friend class LIBERTIFFDataset; + + std::recursive_mutex m_oMutexBlockCache{}; + GDALColorInterp m_eColorInterp = GCI_Undefined; + std::unique_ptr m_poColorTable{}; + bool m_bHasNoData = false; + bool m_bHaveOffsetScale = false; + bool m_bDebugGetLockedBlockRef = false; + double m_dfNoData = 0; + double m_dfScale = 1.0; + double m_dfOffset = 0.0; + std::string m_osUnitType{}; + std::string m_osDescription{}; + + struct ThreadLocalState + { + private: + std::weak_ptr m_validityTest{}; + + public: + explicit ThreadLocalState(const LIBERTIFFBand *band) + : m_validityTest( + cpl::down_cast(band->poDS)->m_validityPtr) + { + } + + inline bool isValid() const + { + return m_validityTest.lock() != nullptr; + } + + GDALDoublePointsCache m_pointsCache{}; + }; + + ThreadLocalState &GetTLSState() const; + + void ReadColorMap(); + + void InitMaskBand(); +}; + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr LIBERTIFFBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pData) +{ + int nXValid, nYValid; + GetActualBlockSize(nBlockXOff, nBlockYOff, &nXValid, &nYValid); + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + int anBand[] = {nBand}; + const int nDTSize = GDALGetDataTypeSizeBytes(eDataType); + return cpl::down_cast(poDS)->IRasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, nXValid, + nYValid, pData, nXValid, nYValid, eDataType, 1, anBand, nDTSize, + static_cast(nDTSize) * nBlockXSize, 0, &sExtraArg); +} + +/************************************************************************/ +/* ReadColorMap() */ +/************************************************************************/ + +void LIBERTIFFBand::ReadColorMap() +{ + auto l_poDS = cpl::down_cast(poDS); + + const auto psTagColorMap = + l_poDS->m_image->tag(LIBERTIFF_NS::TagCode::ColorMap); + if (psTagColorMap && psTagColorMap->type == LIBERTIFF_NS::TagType::Short && + psTagColorMap->count >= 3 && (psTagColorMap->count % 3) == 0 && + psTagColorMap->count == (1U << l_poDS->m_image->bitsPerSample()) * 3U && + !psTagColorMap->invalid_value_offset) + { + bool ok = true; + const auto colorMap = + l_poDS->m_image->readTagAsVector(*psTagColorMap, ok); + if (colorMap.size() == psTagColorMap->count) + { + m_poColorTable = std::make_unique(); + m_eColorInterp = GCI_PaletteIndex; + + // TIFF color maps are in the [0, 65535] range, so some remapping must + // be done to get values in the [0, 255] range, but it is not clear + // how to do that exactly. Since GDAL 2.3.0 we have standardized on + // using a 257 multiplication factor (https://github.com/OSGeo/gdal/commit/eeec5b62e385d53e7f2edaba7b73c7c74bc2af39) + // but other software uses 256 (cf https://github.com/OSGeo/gdal/issues/10310) + // Do a first pass to check if all values are multiples of 256 or 257. + bool bFoundNonZeroEntry = false; + bool bAllValuesMultipleOf256 = true; + bool bAllValuesMultipleOf257 = true; + unsigned short nMaxColor = 0; + int nColorCount = static_cast(psTagColorMap->count / 3); + const auto panRed = colorMap.data(); + const auto panGreen = colorMap.data() + nColorCount; + const auto panBlue = colorMap.data() + 2 * nColorCount; + for (int iColor = 0; iColor < nColorCount; ++iColor) + { + if (panRed[iColor] > 0 || panGreen[iColor] > 0 || + panBlue[iColor] > 0) + { + bFoundNonZeroEntry = true; + } + if ((panRed[iColor] % 256) != 0 || + (panGreen[iColor] % 256) != 0 || + (panBlue[iColor] % 256) != 0) + { + bAllValuesMultipleOf256 = false; + } + if ((panRed[iColor] % 257) != 0 || + (panGreen[iColor] % 257) != 0 || + (panBlue[iColor] % 257) != 0) + { + bAllValuesMultipleOf257 = false; + } + + nMaxColor = std::max(nMaxColor, panRed[iColor]); + nMaxColor = std::max(nMaxColor, panGreen[iColor]); + nMaxColor = std::max(nMaxColor, panBlue[iColor]); + } + + int nColorTableMultiplier; + if (nMaxColor > 0 && nMaxColor < 256) + { + // Bug 1384 - Some TIFF files are generated with color map entry + // values in range 0-255 instead of 0-65535 - try to handle these + // gracefully. + nColorTableMultiplier = 1; + CPLDebug("GTiff", + "TIFF ColorTable seems to be improperly scaled with " + "values all in [0,255] range, fixing up."); + } + else + { + if (!bAllValuesMultipleOf256 && !bAllValuesMultipleOf257) + { + CPLDebug("GTiff", + "The color map contains entries which are not " + "multiple of 256 or 257, so we don't know for " + "sure how to remap them to [0, 255]. Default to " + "using a 257 multiplication factor"); + } + constexpr int DEFAULT_COLOR_TABLE_MULTIPLIER_257 = 257; + nColorTableMultiplier = + (bFoundNonZeroEntry && bAllValuesMultipleOf256) + ? 256 + : DEFAULT_COLOR_TABLE_MULTIPLIER_257; + } + + CPLAssert(nColorTableMultiplier > 0); + CPLAssert(nColorTableMultiplier <= 257); + for (int iColor = nColorCount - 1; iColor >= 0; iColor--) + { + const GDALColorEntry oEntry = { + static_cast(panRed[iColor] / nColorTableMultiplier), + static_cast(panGreen[iColor] / + nColorTableMultiplier), + static_cast(panBlue[iColor] / nColorTableMultiplier), + static_cast( + m_bHasNoData && static_cast(m_dfNoData) == iColor + ? 0 + : 255)}; + + m_poColorTable->SetColorEntry(iColor, &oEntry); + } + } + } +} + +/************************************************************************/ +/* GetTLSState() */ +/************************************************************************/ + +LIBERTIFFBand::ThreadLocalState &LIBERTIFFBand::GetTLSState() const +{ + static thread_local lru11::Cache> + tlsState; + std::shared_ptr value; + if (tlsState.tryGet(this, value)) + { + if (value->isValid()) + return *value.get(); + } + value = std::make_shared(this); + tlsState.insert(this, value); + return *value.get(); +} + +/************************************************************************/ +/* InterpolateAtPoint() */ +/************************************************************************/ + +CPLErr LIBERTIFFBand::InterpolateAtPoint(double dfPixel, double dfLine, + GDALRIOResampleAlg eInterpolation, + double *pdfRealValue, + double *pdfImagValue) const +{ + if (eInterpolation != GRIORA_NearestNeighbour && + eInterpolation != GRIORA_Bilinear && eInterpolation != GRIORA_Cubic && + eInterpolation != GRIORA_CubicSpline) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Only nearest, bilinear, cubic and cubicspline interpolation " + "methods " + "allowed"); + + return CE_Failure; + } + + LIBERTIFFBand *pBand = const_cast(this); + auto &pointsCache = GetTLSState().m_pointsCache; + const bool res = + GDALInterpolateAtPoint(pBand, eInterpolation, pointsCache.cache, + dfPixel, dfLine, pdfRealValue, pdfImagValue); + + return res ? CE_None : CE_Failure; +} + +/************************************************************************/ +/* InitMaskBand() */ +/************************************************************************/ + +void LIBERTIFFBand::InitMaskBand() +{ + auto l_poDS = cpl::down_cast(poDS); + if (m_bHasNoData) + { + nMaskFlags = GMF_NODATA; + poMask.reset(std::make_unique(this)); + } + else if (l_poDS->m_poMaskDS) + { + nMaskFlags = GMF_PER_DATASET; + poMask.reset(l_poDS->m_poMaskDS->GetRasterBand(1), false); + } + else if (l_poDS->m_poAlphaBand && l_poDS->m_poAlphaBand != this) + { + nMaskFlags = GMF_PER_DATASET | GMF_ALPHA; + poMask.reset(l_poDS->m_poAlphaBand, false); + } + else + { + nMaskFlags = GMF_ALL_VALID; + poMask.reset(std::make_unique(this)); + } +} + +/************************************************************************/ +/* GetMetadataItem() */ +/************************************************************************/ + +const char *LIBERTIFFBand::GetMetadataItem(const char *pszName, + const char *pszDomain) +{ + + if (pszName != nullptr && pszDomain != nullptr && EQUAL(pszDomain, "TIFF")) + { + int nBlockXOff = 0; + int nBlockYOff = 0; + + auto l_poDS = cpl::down_cast(poDS); + + if (EQUAL(pszName, "JPEGTABLES")) + { + if (l_poDS->m_jpegTablesOri.empty()) + return nullptr; + char *const pszHex = + CPLBinaryToHex(static_cast(l_poDS->m_jpegTablesOri.size()), + l_poDS->m_jpegTablesOri.data()); + const char *pszReturn = CPLSPrintf("%s", pszHex); + CPLFree(pszHex); + + return pszReturn; + } + + if (EQUAL(pszName, "IFD_OFFSET")) + { + return CPLSPrintf(CPL_FRMT_GUIB, + static_cast(l_poDS->m_image->offset())); + } + + if (sscanf(pszName, "BLOCK_OFFSET_%d_%d", &nBlockXOff, &nBlockYOff) == + 2) + { + if (nBlockXOff < 0 || + nBlockXOff >= DIV_ROUND_UP(nRasterXSize, nBlockXSize) || + nBlockYOff < 0 || + nBlockYOff >= DIV_ROUND_UP(nRasterYSize, nBlockYSize)) + return nullptr; + + uint64_t curStrileIdx = + static_cast(nBlockYOff) * + DIV_ROUND_UP(nRasterXSize, nBlockXSize) + + nBlockXOff; + if (l_poDS->m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate) + { + curStrileIdx += (nBand - 1) * + static_cast( + DIV_ROUND_UP(nRasterXSize, nBlockXSize)) * + DIV_ROUND_UP(nRasterYSize, nBlockYSize); + } + + bool ok = true; + const uint64_t offset = + l_poDS->m_image->strileOffset(curStrileIdx, ok); + if (!offset) + { + return nullptr; + } + + return CPLSPrintf(CPL_FRMT_GUIB, static_cast(offset)); + } + + if (sscanf(pszName, "BLOCK_SIZE_%d_%d", &nBlockXOff, &nBlockYOff) == 2) + { + if (nBlockXOff < 0 || + nBlockXOff >= DIV_ROUND_UP(nRasterXSize, nBlockXSize) || + nBlockYOff < 0 || + nBlockYOff >= DIV_ROUND_UP(nRasterYSize, nBlockYSize)) + return nullptr; + + uint64_t curStrileIdx = + static_cast(nBlockYOff) * + DIV_ROUND_UP(nRasterXSize, nBlockXSize) + + nBlockXOff; + if (l_poDS->m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate) + { + curStrileIdx += (nBand - 1) * + static_cast( + DIV_ROUND_UP(nRasterXSize, nBlockXSize)) * + DIV_ROUND_UP(nRasterYSize, nBlockYSize); + } + + bool ok = true; + const uint64_t size = + l_poDS->m_image->strileByteCount(curStrileIdx, ok); + if (!size) + { + return nullptr; + } + + return CPLSPrintf(CPL_FRMT_GUIB, static_cast(size)); + } + } + return GDALPamRasterBand::GetMetadataItem(pszName, pszDomain); +} + +/************************************************************************/ +/* GetTLSState() */ +/************************************************************************/ + +LIBERTIFFDataset::ThreadLocalState &LIBERTIFFDataset::GetTLSState() const +{ + static thread_local lru11::Cache> + tlsState; + + std::shared_ptr value; + if (tlsState.tryGet(this, value)) + { + if (value->isValid()) + return *value.get(); + } + value = std::make_shared(this); + tlsState.insert(this, value); + return *value.get(); +} + +/************************************************************************/ +/* IRasterIO() */ +/************************************************************************/ + +CPLErr LIBERTIFFDataset::IRasterIO( + GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize, + void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, + int nBandCount, BANDMAP_TYPE panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg) +{ + if (eRWFlag != GF_Read) + return CE_Failure; + + // Try to pass the request to the most appropriate overview dataset. + if (nBufXSize < nXSize && nBufYSize < nYSize) + { + int bTried = FALSE; + const CPLErr eErr = TryOverviewRasterIO( + eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, nBufYSize, + eBufType, nBandCount, panBandMap, nPixelSpace, nLineSpace, + nBandSpace, psExtraArg, &bTried); + if (bTried) + return eErr; + } + + const GDALDataType eNativeDT = papoBands[0]->GetRasterDataType(); + const size_t nNativeDTSize = + static_cast(GDALGetDataTypeSizeBytes(eNativeDT)); + int nBlockXSize, nBlockYSize; + papoBands[0]->GetBlockSize(&nBlockXSize, &nBlockYSize); + + const int iXBlockMin = nXOff / nBlockXSize; + const int iYBlockMin = nYOff / nBlockYSize; + if (nXSize == 1 && nYSize == 1 && nBufXSize == 1 && nBufYSize == 1) + { + ThreadLocalState &tlsState = GetTLSState(); + const double dfNoData = + cpl::down_cast(papoBands[0])->m_dfNoData; + const size_t nXYOffset = + static_cast(nYOff % nBlockYSize) * nBlockXSize + + (nXOff % nBlockXSize); + if (m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate) + { + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + int anBand[] = {panBandMap[iBand]}; + if (!ReadBlock(nullptr, iXBlockMin, iYBlockMin, 1, anBand, + eBufType, nPixelSpace, nLineSpace, nBandSpace)) + { + return CE_Failure; + } + if (tlsState.m_curStrileMissing) + { + GDALCopyWords64(&dfNoData, GDT_Float64, 0, + static_cast(pData) + + iBand * nBandSpace, + eBufType, 0, 1); + } + else + { + GDALCopyWords64(tlsState.m_decompressedBuffer.data() + + nNativeDTSize * nXYOffset, + eNativeDT, 0, + static_cast(pData) + + iBand * nBandSpace, + eBufType, 0, 1); + } + } + } + else + { + if (!ReadBlock(nullptr, iXBlockMin, iYBlockMin, nBandCount, + panBandMap, eBufType, nPixelSpace, nLineSpace, + nBandSpace)) + { + return CE_Failure; + } + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + if (tlsState.m_curStrileMissing) + { + GDALCopyWords64(&dfNoData, GDT_Float64, 0, + static_cast(pData) + + iBand * nBandSpace, + eBufType, 0, 1); + } + else + { + GDALCopyWords64( + tlsState.m_decompressedBuffer.data() + + nNativeDTSize * + (panBandMap[iBand] - 1 + nXYOffset * nBands), + eNativeDT, 0, + static_cast(pData) + iBand * nBandSpace, + eBufType, 0, 1); + } + } + } + return CE_None; + } + + // Check that request is full resolution and aligned on block boundaries + // (with the exception of the right and bottom most blocks that can be + // truncated) + if (nXSize != nBufXSize || nYSize != nBufYSize || + (nXOff % nBlockXSize) != 0 || (nYOff % nBlockYSize) != 0 || + !(nXOff + nXSize == nRasterXSize || (nBufXSize % nBlockXSize) == 0) || + !(nYOff + nYSize == nRasterYSize || (nBufYSize % nBlockYSize) == 0)) + { + const int nXOffMod = (nXOff / nBlockXSize) * nBlockXSize; + const int nYOffMod = (nYOff / nBlockYSize) * nBlockYSize; + const int nXOff2Mod = static_cast(std::min( + static_cast(nRasterXSize), + static_cast(DIV_ROUND_UP(nXOff + nXSize, nBlockXSize)) * + nBlockXSize)); + const int nYOff2Mod = static_cast(std::min( + static_cast(nRasterYSize), + static_cast(DIV_ROUND_UP(nYOff + nYSize, nBlockYSize)) * + nBlockYSize)); + const int nXSizeMod = nXOff2Mod - nXOffMod; + const int nYSizeMod = nYOff2Mod - nYOffMod; + std::vector &abyTmp = GetTLSState().m_abyIRasterIOBuffer; + + try + { + if (nNativeDTSize * nBandCount > + std::numeric_limits::max() / + (static_cast(nXSizeMod) * nYSizeMod)) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating temporary buffer"); + return CE_Failure; + } + const size_t nSize = + nNativeDTSize * nBandCount * nXSizeMod * nYSizeMod; + if (abyTmp.size() < nSize) + abyTmp.resize(nSize); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating temporary buffer"); + return CE_Failure; + } + + { + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + + if (IRasterIO(GF_Read, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, + abyTmp.data(), nXSizeMod, nYSizeMod, eNativeDT, + nBandCount, panBandMap, + GDALGetDataTypeSizeBytes(eNativeDT), + nNativeDTSize * nXSizeMod, + nNativeDTSize * nXSizeMod * nYSizeMod, + &sExtraArg) != CE_None) + { + return CE_Failure; + } + } + + auto poMEMDS = std::unique_ptr(MEMDataset::Create( + "", nXSizeMod, nYSizeMod, 0, GDT_Unknown, nullptr)); + if (!poMEMDS) + { + return CE_Failure; + } + for (int i = 0; i < nBandCount; ++i) + { + GByte *pabyData = + abyTmp.data() + i * nNativeDTSize * nXSizeMod * nYSizeMod; + GDALRasterBandH hMEMBand = MEMCreateRasterBandEx( + poMEMDS.get(), i + 1, pabyData, eNativeDT, nNativeDTSize, + nNativeDTSize * nXSizeMod, false); + poMEMDS->AddMEMBand(hMEMBand); + } + + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + // cppcheck-suppress redundantAssignment + sExtraArg.eResampleAlg = psExtraArg->eResampleAlg; + sExtraArg.bFloatingPointWindowValidity = + psExtraArg->bFloatingPointWindowValidity; + if (sExtraArg.bFloatingPointWindowValidity) + { + sExtraArg.dfXOff = psExtraArg->dfXOff - nXOffMod; + sExtraArg.dfYOff = psExtraArg->dfYOff - nYOffMod; + sExtraArg.dfXSize = psExtraArg->dfXSize; + sExtraArg.dfYSize = psExtraArg->dfYSize; + } + return poMEMDS->RasterIO(GF_Read, nXOff - nXOffMod, nYOff - nYOffMod, + nXSize, nYSize, pData, nBufXSize, nBufYSize, + eBufType, nBandCount, nullptr, nPixelSpace, + nLineSpace, nBandSpace, &sExtraArg); + } + + const int iYBlockMax = DIV_ROUND_UP(nYOff + nBufYSize, nBlockYSize); + const int iXBlockMax = DIV_ROUND_UP(nXOff + nBufXSize, nBlockXSize); + + // Could be parallelized + for (int iYBlock = iYBlockMin, iY = 0; iYBlock < iYBlockMax; + ++iYBlock, ++iY) + { + for (int iXBlock = iXBlockMin, iX = 0; iXBlock < iXBlockMax; + ++iXBlock, ++iX) + { + if (m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate) + { + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + int anBand[] = {panBandMap[iBand]}; + if (!ReadBlock(static_cast(pData) + + iY * nLineSpace * nBlockYSize + + iX * nPixelSpace * nBlockXSize + + iBand * nBandSpace, + iXBlock, iYBlock, 1, anBand, eBufType, + nPixelSpace, nLineSpace, nBandSpace)) + { + return CE_Failure; + } + } + } + else + { + if (!ReadBlock(static_cast(pData) + + iY * nLineSpace * nBlockYSize + + iX * nPixelSpace * nBlockXSize, + iXBlock, iYBlock, nBandCount, panBandMap, + eBufType, nPixelSpace, nLineSpace, nBandSpace)) + { + return CE_Failure; + } + } + } + } + + return CE_None; +} + +/************************************************************************/ +/* HorizPredictorDecode() */ +/************************************************************************/ + +template +CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void +HorizPredictorDecode1Component(void *bufferIn, size_t nPixelCount) +{ + T *buffer = static_cast(bufferIn); + constexpr T mask = std::numeric_limits::max(); + U acc = buffer[0]; + size_t i = 1; + for (; i + 3 < nPixelCount; i += 4) + { + acc += buffer[i]; + buffer[i] = static_cast(acc & mask); + acc += buffer[i + 1]; + buffer[i + 1] = static_cast(acc & mask); + acc += buffer[i + 2]; + buffer[i + 2] = static_cast(acc & mask); + acc += buffer[i + 3]; + buffer[i + 3] = static_cast(acc & mask); + } + for (; i < nPixelCount; ++i) + { + acc += buffer[i]; + buffer[i] = static_cast(acc & mask); + } +} + +template +CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void +HorizPredictorDecode2Components(void *bufferIn, size_t nPixelCount) +{ + T *buffer = static_cast(bufferIn); + constexpr T mask = std::numeric_limits::max(); + U acc0 = buffer[0]; + U acc1 = buffer[1]; + for (size_t i = 1; i < nPixelCount; ++i) + { + acc0 += buffer[i * 2 + 0]; + acc1 += buffer[i * 2 + 1]; + buffer[i * 2 + 0] = static_cast(acc0 & mask); + buffer[i * 2 + 1] = static_cast(acc1 & mask); + } +} + +template +CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void +HorizPredictorDecode3Components(void *bufferIn, size_t nPixelCount) +{ + T *buffer = static_cast(bufferIn); + constexpr T mask = std::numeric_limits::max(); + U acc0 = buffer[0]; + U acc1 = buffer[1]; + U acc2 = buffer[2]; + for (size_t i = 1; i < nPixelCount; ++i) + { + acc0 += buffer[i * 3 + 0]; + acc1 += buffer[i * 3 + 1]; + acc2 += buffer[i * 3 + 2]; + buffer[i * 3 + 0] = static_cast(acc0 & mask); + buffer[i * 3 + 1] = static_cast(acc1 & mask); + buffer[i * 3 + 2] = static_cast(acc2 & mask); + } +} + +template +CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void +HorizPredictorDecode4Components(void *bufferIn, size_t nPixelCount) +{ + T *buffer = static_cast(bufferIn); + constexpr T mask = std::numeric_limits::max(); + U acc0 = buffer[0]; + U acc1 = buffer[1]; + U acc2 = buffer[2]; + U acc3 = buffer[3]; + for (size_t i = 1; i < nPixelCount; ++i) + { + acc0 += buffer[i * 4 + 0]; + acc1 += buffer[i * 4 + 1]; + acc2 += buffer[i * 4 + 2]; + acc3 += buffer[i * 4 + 3]; + buffer[i * 4 + 0] = static_cast(acc0 & mask); + buffer[i * 4 + 1] = static_cast(acc1 & mask); + buffer[i * 4 + 2] = static_cast(acc2 & mask); + buffer[i * 4 + 3] = static_cast(acc3 & mask); + } +} + +template +CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static void +HorizPredictorDecode(void *bufferIn, size_t nPixelCount, + int nComponentsPerPixel) +{ + static_assert(std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v); + + if (nComponentsPerPixel == 1) + { + // cppcheck-suppress duplicateBranch + if constexpr (sizeof(T) < sizeof(uint64_t)) + { + HorizPredictorDecode1Component(bufferIn, nPixelCount); + } + else + { + HorizPredictorDecode1Component(bufferIn, nPixelCount); + } + } + else if (nComponentsPerPixel == 2) + { + // cppcheck-suppress duplicateBranch + if constexpr (sizeof(T) < sizeof(uint64_t)) + { + HorizPredictorDecode2Components(bufferIn, nPixelCount); + } + else + { + HorizPredictorDecode2Components(bufferIn, nPixelCount); + } + } + else if (nComponentsPerPixel == 3) + { + // cppcheck-suppress duplicateBranch + if constexpr (sizeof(T) < sizeof(uint64_t)) + { + HorizPredictorDecode3Components(bufferIn, nPixelCount); + } + else + { + HorizPredictorDecode3Components(bufferIn, nPixelCount); + } + } + else if (nComponentsPerPixel == 4) + { + // cppcheck-suppress duplicateBranch + if constexpr (sizeof(T) < sizeof(uint64_t)) + { + HorizPredictorDecode4Components(bufferIn, nPixelCount); + } + else + { + HorizPredictorDecode4Components(bufferIn, nPixelCount); + } + } + else + { + T *buffer = static_cast(bufferIn); + constexpr T mask = std::numeric_limits::max(); + for (size_t i = 1; i < nPixelCount; ++i) + { + for (int j = 0; j < nComponentsPerPixel; j++) + { + buffer[i * nComponentsPerPixel + j] = + static_cast((buffer[i * nComponentsPerPixel + j] + + buffer[(i - 1) * nComponentsPerPixel + j]) & + mask); + } + } + } +} + +/************************************************************************/ +/* FloatingPointHorizPredictorDecode() */ +/************************************************************************/ + +template +CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW static bool +FloatingPointHorizPredictorDecode(std::vector &tmpBuffer, + void *bufferIn, size_t nPixelCount, + int nComponentsPerPixel) +{ + uint8_t *buffer = static_cast(bufferIn); + HorizPredictorDecode(buffer, nPixelCount * sizeof(T), + nComponentsPerPixel); + + const size_t tmpBufferSize = nPixelCount * nComponentsPerPixel * sizeof(T); + if (tmpBuffer.size() < tmpBufferSize) + { + try + { + tmpBuffer.resize(tmpBufferSize); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory in FloatingPointHorizPredictorDecode()"); + return false; + } + } + memcpy(tmpBuffer.data(), buffer, tmpBufferSize); + constexpr uint32_t bytesPerWords = static_cast(sizeof(T)); + const size_t wordCount = nPixelCount * nComponentsPerPixel; + for (size_t iWord = 0; iWord < wordCount; iWord++) + { + for (uint32_t iByte = 0; iByte < bytesPerWords; iByte++) + { +#ifdef CPL_MSB + buffer[bytesPerWords * iWord + iByte] = + tmpBuffer[iByte * wordCount + iWord]; +#else + buffer[bytesPerWords * iWord + iByte] = + tmpBuffer[(bytesPerWords - iByte - 1) * wordCount + iWord]; +#endif + } + } + return true; +} + +/************************************************************************/ +/* ExtractBitAndConvertTo255() */ +/************************************************************************/ + +#if defined(__GNUC__) || defined(_MSC_VER) +// Signedness of char implementation dependent, so be explicit. +// Assumes 2-complement integer types and sign extension of right shifting +// GCC guarantees such: +// https://gcc.gnu.org/onlinedocs/gcc/Integers-implementation.html#Integers-implementation +static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit) +{ + return static_cast(static_cast(byVal << (7 - nBit)) >> + 7); +} +#else +// Portable way +static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit) +{ + return (byVal & (1 << nBit)) ? 255 : 0; +} +#endif + +/************************************************************************/ +/* ReadBlock() */ +/************************************************************************/ + +bool LIBERTIFFDataset::ReadBlock(GByte *pabyBlockData, int nBlockXOff, + int nBlockYOff, int nBandCount, + BANDMAP_TYPE panBandMap, GDALDataType eBufType, + GSpacing nPixelSpace, GSpacing nLineSpace, + GSpacing nBandSpace) const +{ + uint64_t offset = 0; + size_t size = 0; + const bool bSeparate = m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate; + + ThreadLocalState &tlsState = GetTLSState(); + + const int iBandTIFFFirst = bSeparate ? panBandMap[0] - 1 : 0; + uint64_t curStrileIdx; + if (m_image->isTiled()) + { + bool ok = true; + curStrileIdx = m_image->tileCoordinateToIdx(nBlockXOff, nBlockYOff, + iBandTIFFFirst, ok); + } + else + { + if (bSeparate) + curStrileIdx = + nBlockYOff + DIV_ROUND_UP(m_image->height(), + m_image->rowsPerStripSanitized()) * + iBandTIFFFirst; + else + curStrileIdx = nBlockYOff; + } + if (curStrileIdx != tlsState.m_curStrileIdx) + { + bool ok = true; + offset = curStrileIdx < m_tileOffsets.size() + ? m_tileOffsets[static_cast(curStrileIdx)] + : curStrileIdx < m_tileOffsets64.size() + ? m_tileOffsets64[static_cast(curStrileIdx)] + : m_image->strileOffset(curStrileIdx, ok); + if (!ok) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot read strile offset"); + return false; + } + const uint64_t size64 = + curStrileIdx < m_tileByteCounts.size() + ? m_tileByteCounts[static_cast(curStrileIdx)] + : m_image->strileByteCount(curStrileIdx, ok); + if (!ok) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot read strile size"); + return false; + } + + if constexpr (sizeof(size_t) < sizeof(uint64_t)) + { + if (size64 > std::numeric_limits::max()) + { + CPLError(CE_Failure, CPLE_NotSupported, "Too large strile"); + return false; + } + } + size = static_cast(size64); + // Avoid doing non-sensical memory allocations + constexpr size_t THRESHOLD_CHECK_FILE_SIZE = 10 * 1024 * 1024; + if (size > THRESHOLD_CHECK_FILE_SIZE && + size > m_image->readContext()->size()) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Strile size larger than file size"); + return false; + } + } + + const GDALDataType eNativeDT = papoBands[0]->GetRasterDataType(); + int nBlockXSize, nBlockYSize; + papoBands[0]->GetBlockSize(&nBlockXSize, &nBlockYSize); + const int nBlockActualXSize = + std::min(nBlockXSize, nRasterXSize - nBlockXOff * nBlockXSize); + const int nBlockActualYSize = + std::min(nBlockYSize, nRasterYSize - nBlockYOff * nBlockYSize); + + // Sparse block? + if ((curStrileIdx != tlsState.m_curStrileIdx && size == 0) || + (curStrileIdx == tlsState.m_curStrileIdx && + tlsState.m_curStrileMissing)) + { + if (pabyBlockData) + { + const double dfNoData = + cpl::down_cast(papoBands[0])->m_dfNoData; + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + for (int iY = 0; iY < nBlockActualYSize; ++iY) + { + GDALCopyWords64(&dfNoData, GDT_Float64, 0, + pabyBlockData + iBand * nBandSpace + + iY * nLineSpace, + eBufType, static_cast(nPixelSpace), + nBlockActualXSize); + } + } + } + + tlsState.m_curStrileIdx = curStrileIdx; + tlsState.m_curStrileMissing = true; + return true; + } + + std::vector &abyDecompressedStrile = tlsState.m_decompressedBuffer; + const size_t nNativeDTSize = + static_cast(GDALGetDataTypeSizeBytes(eNativeDT)); + + if (curStrileIdx != tlsState.m_curStrileIdx) + { + std::vector &bufferForOneBitExpansion = + tlsState.m_bufferForOneBitExpansion; + + // Overflow in multiplication checked in Open() method + const int nComponentsPerPixel = bSeparate ? 1 : nBands; + const size_t nActualPixelCount = + static_cast(m_image->isTiled() ? nBlockYSize + : nBlockActualYSize) * + nBlockXSize; + const int nLineSizeBytes = + m_image->bitsPerSample() == 1 ? (nBlockXSize + 7) / 8 : nBlockXSize; + const size_t nActualUncompressedSize = + nNativeDTSize * + static_cast(m_image->isTiled() ? nBlockYSize + : nBlockActualYSize) * + nLineSizeBytes * nComponentsPerPixel; + + // Allocate buffer for decompressed strile + if (abyDecompressedStrile.empty()) + { + const size_t nMaxUncompressedSize = + nNativeDTSize * nBlockXSize * nBlockYSize * nComponentsPerPixel; + try + { + abyDecompressedStrile.resize(nMaxUncompressedSize); + if (m_image->bitsPerSample() == 1) + bufferForOneBitExpansion.resize(nMaxUncompressedSize); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating temporary buffer"); + return false; + } + } + + if (m_image->compression() != LIBERTIFF_NS::Compression::None) + { + std::vector &abyCompressedStrile = + tlsState.m_compressedBuffer; + if (size > 128 && size / 16 > nActualUncompressedSize) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Compressed strile size is much larger than " + "uncompressed size"); + return false; + } + if (abyCompressedStrile.size() < size + m_jpegTables.size()) + { + try + { + abyCompressedStrile.resize(size + m_jpegTables.size()); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating temporary buffer"); + return false; + } + } + + bool ok = true; + m_image->readContext()->read(offset, size, + abyCompressedStrile.data(), ok); + if (!ok) + { + CPLError(CE_Failure, CPLE_FileIO, + "Cannot read strile from disk"); + return false; + } + + if (!tlsState.m_tiff.tif_decodestrip) + { + if (m_image->compression() == LIBERTIFF_NS::Compression::LZW) + { + TIFFInitLZW(&tlsState.m_tiff, m_image->compression()); + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::PackBits) + { + TIFFInitPackBits(&tlsState.m_tiff, m_image->compression()); + } +#ifdef LERC_SUPPORT + else if (m_image->compression() == + LIBERTIFF_NS::Compression::LERC) + { + TIFFInitLERC(&tlsState.m_tiff, m_image->compression()); + LERCState *sp = + reinterpret_cast(tlsState.m_tiff.tif_data); + sp->lerc_version = m_lercVersion; + sp->additional_compression = m_lercAdditionalCompression; + } +#endif + + if (tlsState.m_tiff.tif_decodestrip) + { + tlsState.m_tiff.tif_dir.td_sampleformat = + static_cast(m_image->sampleFormat()); + tlsState.m_tiff.tif_dir.td_bitspersample = + static_cast(m_image->bitsPerSample()); + if (m_image->isTiled()) + { + tlsState.m_tiff.tif_flags = TIFF_ISTILED; + tlsState.m_tiff.tif_dir.td_tilewidth = + m_image->tileWidth(); + tlsState.m_tiff.tif_dir.td_tilelength = + m_image->tileHeight(); + } + else + { + tlsState.m_tiff.tif_dir.td_imagewidth = + m_image->width(); + tlsState.m_tiff.tif_dir.td_imagelength = + m_image->height(); + tlsState.m_tiff.tif_dir.td_rowsperstrip = + m_image->rowsPerStripSanitized(); + } + tlsState.m_tiff.tif_dir.td_samplesperpixel = + static_cast(m_image->samplesPerPixel()); + tlsState.m_tiff.tif_dir.td_planarconfig = + static_cast(m_image->planarConfiguration()); + if (m_extraSamples.size() < 65536) + { + tlsState.m_tiff.tif_dir.td_extrasamples = + static_cast(m_extraSamples.size()); + tlsState.m_tiff.tif_dir.td_sampleinfo = + const_cast(m_extraSamples.data()); + } + } + } + + if (tlsState.m_tiff.tif_decodestrip) + { + tlsState.m_tiff.tif_row = nBlockYOff * nBlockYSize; + tlsState.m_tiff.tif_rawcc = size; + tlsState.m_tiff.tif_rawdata = abyCompressedStrile.data(); + tlsState.m_tiff.tif_rawcp = tlsState.m_tiff.tif_rawdata; + if ((tlsState.m_tiff.tif_predecode && + tlsState.m_tiff.tif_predecode(&tlsState.m_tiff, 0) == 0) || + tlsState.m_tiff.tif_decodestrip( + &tlsState.m_tiff, abyDecompressedStrile.data(), + nActualUncompressedSize, 0) == 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Decompression failed"); + return false; + } + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::JPEG || + m_image->compression() == + LIBERTIFF_NS::Compression::WEBP || + m_image->compression() == LIBERTIFF_NS::Compression::JXL || + m_image->compression() == + LIBERTIFF_NS::Compression::JXL_DNG_1_7) + { + size_t blobSize = size; + const char *drvName = + m_image->compression() == LIBERTIFF_NS::Compression::JPEG + ? "JPEG" + : m_image->compression() == LIBERTIFF_NS::Compression::WEBP + ? "WEBP" + : "JPEGXL"; + if (m_image->compression() == LIBERTIFF_NS::Compression::JPEG && + size > 2 && !m_jpegTables.empty()) + { + // Insert JPEG tables into JPEG blob + memmove(abyCompressedStrile.data() + 2 + + m_jpegTables.size(), + abyCompressedStrile.data() + 2, size - 2); + memcpy(abyCompressedStrile.data() + 2, m_jpegTables.data(), + m_jpegTables.size()); + blobSize += m_jpegTables.size(); + } + const std::string osTmpFilename = VSIMemGenerateHiddenFilename( + std::string("tmp.").append(drvName).c_str()); + VSIFCloseL(VSIFileFromMemBuffer( + osTmpFilename.c_str(), abyCompressedStrile.data(), blobSize, + /* bTakeOwnership = */ false)); + const char *const apszAllowedDrivers[] = {drvName, nullptr}; + + CPLConfigOptionSetter oJPEGtoRGBSetter( + "GDAL_JPEG_TO_RGB", + m_image->compression() == LIBERTIFF_NS::Compression::JPEG && + m_image->samplesPerPixel() == 4 && + m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Contiguous + ? "NO" + : "YES", + false); + + auto poTmpDS = std::unique_ptr(GDALDataset::Open( + osTmpFilename.c_str(), GDAL_OF_RASTER | GDAL_OF_INTERNAL, + apszAllowedDrivers, nullptr, nullptr)); + VSIUnlink(osTmpFilename.c_str()); + if (!poTmpDS) + { + CPLError(CE_Failure, CPLE_AppDefined, "Not a %s blob", + drvName); + return false; + } + if (poTmpDS->GetRasterCount() != nComponentsPerPixel || + poTmpDS->GetRasterXSize() != nBlockXSize || + poTmpDS->GetRasterYSize() != + (m_image->isTiled() ? nBlockYSize : nBlockActualYSize)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s blob has no expected dimensions (%dx%d " + "whereas %dx%d expected) or band count (%d " + "whereas %d expected)", + drvName, poTmpDS->GetRasterXSize(), + poTmpDS->GetRasterYSize(), nBlockXSize, + m_image->isTiled() ? nBlockYSize + : nBlockActualYSize, + poTmpDS->GetRasterCount(), nComponentsPerPixel); + return false; + } + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if (poTmpDS->RasterIO( + GF_Read, 0, 0, poTmpDS->GetRasterXSize(), + poTmpDS->GetRasterYSize(), abyDecompressedStrile.data(), + poTmpDS->GetRasterXSize(), poTmpDS->GetRasterYSize(), + eNativeDT, poTmpDS->GetRasterCount(), nullptr, + nNativeDTSize * nComponentsPerPixel, + nNativeDTSize * nComponentsPerPixel * nBlockXSize, + nNativeDTSize, &sExtraArg) != CE_None) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Decompression failed"); + return false; + } + } + else + { + CPLAssert(m_decompressor); + void *output_data = abyDecompressedStrile.data(); + size_t output_size = nActualUncompressedSize; + if (!m_decompressor->pfnFunc( + abyCompressedStrile.data(), size, &output_data, + &output_size, nullptr, m_decompressor->user_data) || + output_size != nActualUncompressedSize) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Decompression failed"); + return false; + } + CPLAssert(output_data == abyDecompressedStrile.data()); + } + } + else + { + if (size != nActualUncompressedSize) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Strile size != expected size"); + return false; + } + + bool ok = true; + m_image->readContext()->read(offset, size, + abyDecompressedStrile.data(), ok); + if (!ok) + { + CPLError(CE_Failure, CPLE_FileIO, + "Cannot read strile from disk"); + return false; + } + } + + if (m_image->bitsPerSample() == 1) + { + const GByte *CPL_RESTRICT pabySrc = abyDecompressedStrile.data(); + const GByte val = m_bExpand1To255 ? 255 : 1; + GByte *CPL_RESTRICT pabyDst = bufferForOneBitExpansion.data(); + for (int iY = 0; iY < nBlockActualYSize; ++iY) + { + int iX = 0; + if (m_bExpand1To255) + { + for (; iX + 7 < nBlockXSize; + iX += 8, ++pabySrc, pabyDst += 8) + { + const GByte srcByte = *pabySrc; + pabyDst[0] = ExtractBitAndConvertTo255(srcByte, 7); + pabyDst[1] = ExtractBitAndConvertTo255(srcByte, 6); + pabyDst[2] = ExtractBitAndConvertTo255(srcByte, 5); + pabyDst[3] = ExtractBitAndConvertTo255(srcByte, 4); + pabyDst[4] = ExtractBitAndConvertTo255(srcByte, 3); + pabyDst[5] = ExtractBitAndConvertTo255(srcByte, 2); + pabyDst[6] = ExtractBitAndConvertTo255(srcByte, 1); + pabyDst[7] = ExtractBitAndConvertTo255(srcByte, 0); + } + } + else + { + for (; iX + 7 < nBlockXSize; + iX += 8, ++pabySrc, pabyDst += 8) + { + const int srcByte = *pabySrc; + pabyDst[0] = (srcByte >> 7) & 1; + pabyDst[1] = (srcByte >> 6) & 1; + pabyDst[2] = (srcByte >> 5) & 1; + pabyDst[3] = (srcByte >> 4) & 1; + pabyDst[4] = (srcByte >> 3) & 1; + pabyDst[5] = (srcByte >> 2) & 1; + pabyDst[6] = (srcByte >> 1) & 1; + pabyDst[7] = (srcByte >> 0) & 1; + } + } + if (iX < nBlockXSize) + { + for (; iX < nBlockXSize; ++iX, ++pabyDst) + { + *pabyDst = (*pabySrc & (0x80 >> (iX % 8))) ? val : 0; + } + ++pabySrc; + } + } + + std::swap(abyDecompressedStrile, bufferForOneBitExpansion); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::None || + m_image->compression() == LIBERTIFF_NS::Compression::LZW || + m_decompressor) + { + if (m_image->readContext()->mustByteSwap() && + m_image->predictor() != 3) + { + if (GDALDataTypeIsComplex(eNativeDT)) + { + GDALSwapWordsEx(abyDecompressedStrile.data(), + static_cast(nNativeDTSize) / 2, + nActualPixelCount * nComponentsPerPixel * 2, + static_cast(nNativeDTSize) / 2); + } + else + { + GDALSwapWordsEx(abyDecompressedStrile.data(), + static_cast(nNativeDTSize), + nActualPixelCount * nComponentsPerPixel, + static_cast(nNativeDTSize)); + } + } + + if (m_image->predictor() == 2) + { + for (int iY = 0; iY < nBlockActualYSize; ++iY) + { + auto ptr = + abyDecompressedStrile.data() + + nNativeDTSize * iY * nBlockXSize * nComponentsPerPixel; + if (nNativeDTSize == sizeof(uint8_t)) + { + HorizPredictorDecode(ptr, nBlockXSize, + nComponentsPerPixel); + } + else if (nNativeDTSize == sizeof(uint16_t)) + { + HorizPredictorDecode(ptr, nBlockXSize, + nComponentsPerPixel); + } + else if (nNativeDTSize == sizeof(uint32_t)) + { + HorizPredictorDecode(ptr, nBlockXSize, + nComponentsPerPixel); + } + else if (nNativeDTSize == sizeof(uint64_t)) + { + HorizPredictorDecode(ptr, nBlockXSize, + nComponentsPerPixel); + } + else + { + CPLAssert(false); + } + } + } + else if (m_image->predictor() == 3) + { + for (int iY = 0; iY < nBlockActualYSize; ++iY) + { + auto ptr = + abyDecompressedStrile.data() + + nNativeDTSize * iY * nBlockXSize * nComponentsPerPixel; + bool ok = true; + if (nNativeDTSize == sizeof(uint32_t)) + { + ok = FloatingPointHorizPredictorDecode( + tlsState + .m_floatingPointHorizPredictorDecodeTmpBuffer, + ptr, nBlockXSize, nComponentsPerPixel); + } + else if (nNativeDTSize == sizeof(uint64_t)) + { + ok = FloatingPointHorizPredictorDecode( + tlsState + .m_floatingPointHorizPredictorDecodeTmpBuffer, + ptr, nBlockXSize, nComponentsPerPixel); + } + else + { + CPLAssert(false); + } + if (!ok) + return false; + } + } + } + } + + // Copy decompress strile into user buffer + if (pabyBlockData) + { + const auto IsContiguousBandMap = [nBandCount, panBandMap]() + { + for (int i = 0; i < nBandCount; ++i) + { + if (panBandMap[i] != i + 1) + return false; + } + return true; + }; + + const int nBufTypeSize = GDALGetDataTypeSizeBytes(eBufType); + if (!bSeparate && nBands > 1 && nBands == nBandCount && + nBufTypeSize == nPixelSpace && IsContiguousBandMap()) + { + // Optimization: reading a pixel-interleaved buffer into a band-interleaved buffer + std::vector &apabyDest = tlsState.m_apabyDest; + apabyDest.resize(nBands); + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + apabyDest[iBand] = pabyBlockData + iBand * nBandSpace; + } + for (int iY = 0; iY < nBlockActualYSize; ++iY) + { + if (iY > 0) + { + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + apabyDest[iBand] = + static_cast(apabyDest[iBand]) + nLineSpace; + } + } + GDALDeinterleave(abyDecompressedStrile.data() + + nNativeDTSize * iY * nBlockXSize * nBands, + eNativeDT, nBands, apabyDest.data(), eBufType, + nBlockActualXSize); + } + } + else if (!bSeparate && nBands == nBandCount && + nBufTypeSize == nBandSpace && + nPixelSpace == nBandSpace * nBandCount && + IsContiguousBandMap()) + { + // Optimization reading a pixel-interleaved buffer into a pixel-interleaved buffer + for (int iY = 0; iY < nBlockActualYSize; ++iY) + { + GDALCopyWords64( + abyDecompressedStrile.data() + + nNativeDTSize * iY * nBlockXSize * nBands, + eNativeDT, static_cast(nNativeDTSize), + pabyBlockData + iY * nLineSpace, eBufType, nBufTypeSize, + static_cast( + static_cast(nBlockActualXSize) * nBands)); + } + } + else + { + // General case + const int nSrcPixels = bSeparate ? 1 : nBands; + for (int iBand = 0; iBand < nBandCount; ++iBand) + { + const int iSrcBand = bSeparate ? 0 : panBandMap[iBand] - 1; + for (int iY = 0; iY < nBlockActualYSize; ++iY) + { + GDALCopyWords64( + abyDecompressedStrile.data() + + nNativeDTSize * + (iY * nBlockXSize * nSrcPixels + iSrcBand), + eNativeDT, static_cast(nSrcPixels * nNativeDTSize), + pabyBlockData + iBand * nBandSpace + iY * nLineSpace, + eBufType, nBufTypeSize, nBlockActualXSize); + } + } + } + } + + tlsState.m_curStrileIdx = curStrileIdx; + tlsState.m_curStrileMissing = false; + + return true; +} + +/************************************************************************/ +/* Identify() */ +/************************************************************************/ + +/* static */ int LIBERTIFFDataset::Identify(GDALOpenInfo *poOpenInfo) +{ + return poOpenInfo->eAccess != GA_Update && + (STARTS_WITH_CI(poOpenInfo->pszFilename, "GTIFF_DIR:") || + (poOpenInfo->fpL && poOpenInfo->nHeaderBytes >= 8 && + (((poOpenInfo->pabyHeader[0] == 'I' && + poOpenInfo->pabyHeader[1] == 'I') && + ((poOpenInfo->pabyHeader[2] == 0x2A && + poOpenInfo->pabyHeader[3] == 0) || + (poOpenInfo->pabyHeader[2] == 0x2B && + poOpenInfo->pabyHeader[3] == 0))) || + ((poOpenInfo->pabyHeader[0] == 'M' && + poOpenInfo->pabyHeader[1] == 'M') && + ((poOpenInfo->pabyHeader[2] == 0 && + poOpenInfo->pabyHeader[3] == 0x2A) || + (poOpenInfo->pabyHeader[2] == 0 && + poOpenInfo->pabyHeader[3] == 0x2B)))))); +} + +/************************************************************************/ +/* ComputeGDALDataType() */ +/************************************************************************/ + +GDALDataType LIBERTIFFDataset::ComputeGDALDataType() const +{ + + GDALDataType eDT = GDT_Unknown; + + switch (m_image->sampleFormat()) + { + case LIBERTIFF_NS::SampleFormat::UnsignedInt: + { + if (m_image->bitsPerSample() == 1 && + (m_image->samplesPerPixel() == 1 || + m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate)) + { + eDT = GDT_Byte; + } + else if (m_image->bitsPerSample() == 8) + eDT = GDT_Byte; + else if (m_image->bitsPerSample() == 16) + eDT = GDT_UInt16; + else if (m_image->bitsPerSample() == 32) + eDT = GDT_UInt32; + else if (m_image->bitsPerSample() == 64) + eDT = GDT_UInt64; + break; + } + + case LIBERTIFF_NS::SampleFormat::SignedInt: + { + if (m_image->bitsPerSample() == 8) + eDT = GDT_Int8; + else if (m_image->bitsPerSample() == 16) + eDT = GDT_Int16; + else if (m_image->bitsPerSample() == 32) + eDT = GDT_Int32; + else if (m_image->bitsPerSample() == 64) + eDT = GDT_Int64; + break; + } + + case LIBERTIFF_NS::SampleFormat::IEEEFP: + { + if (m_image->bitsPerSample() == 32) + eDT = GDT_Float32; + else if (m_image->bitsPerSample() == 64) + eDT = GDT_Float64; + break; + } + + case LIBERTIFF_NS::SampleFormat::ComplexInt: + { + if (m_image->bitsPerSample() == 32) + eDT = GDT_CInt16; + else if (m_image->bitsPerSample() == 64) + eDT = GDT_CInt32; + break; + } + + case LIBERTIFF_NS::SampleFormat::ComplexIEEEFP: + { + if (m_image->bitsPerSample() == 64) + eDT = GDT_CFloat32; + else if (m_image->bitsPerSample() == 128) + eDT = GDT_CFloat64; + break; + } + + default: + break; + } + + if (m_image->bitsPerSample() == 12 && + m_image->compression() == LIBERTIFF_NS::Compression::JPEG) + { + auto poJPEGDrv = GetGDALDriverManager()->GetDriverByName("JPEG"); + if (poJPEGDrv) + { + const char *pszJPEGDataTypes = + poJPEGDrv->GetMetadataItem(GDAL_DMD_CREATIONDATATYPES); + if (pszJPEGDataTypes && strstr(pszJPEGDataTypes, "UInt16")) + eDT = GDT_UInt16; + } + } + + return eDT; +} + +/************************************************************************/ +/* ProcessCompressionMethod() */ +/************************************************************************/ + +bool LIBERTIFFDataset::ProcessCompressionMethod() +{ + if (m_image->compression() == LIBERTIFF_NS::Compression::PackBits) + { + GDALDataset::SetMetadataItem("COMPRESSION", "PACKBITS", + "IMAGE_STRUCTURE"); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::Deflate || + m_image->compression() == LIBERTIFF_NS::Compression::LegacyDeflate) + { + m_decompressor = CPLGetDecompressor("zlib"); + GDALDataset::SetMetadataItem("COMPRESSION", "DEFLATE", + "IMAGE_STRUCTURE"); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::ZSTD) + { + m_decompressor = CPLGetDecompressor("zstd"); + if (!m_decompressor) + { + ReportError(CE_Failure, CPLE_NotSupported, + "Compression = ZSTD unhandled because GDAL " + "has not been built against libzstd"); + return false; + } + GDALDataset::SetMetadataItem("COMPRESSION", "ZSTD", "IMAGE_STRUCTURE"); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::LZMA) + { + m_decompressor = CPLGetDecompressor("lzma"); + if (!m_decompressor) + { + ReportError(CE_Failure, CPLE_NotSupported, + "Compression = LZMA unhandled because GDAL " + "has not been built against liblzma"); + return false; + } + GDALDataset::SetMetadataItem("COMPRESSION", "LZMA", "IMAGE_STRUCTURE"); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::LZW) + { + GDALDataset::SetMetadataItem("COMPRESSION", "LZW", "IMAGE_STRUCTURE"); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::JPEG) + { + if (!GDALGetDriverByName("JPEG")) + { + ReportError( + CE_Failure, CPLE_NotSupported, + "Compression = JPEG not supported because JPEG driver missing"); + return false; + } + if (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::YCbCr && + m_image->samplesPerPixel() == 3) + { + GDALDataset::SetMetadataItem("SOURCE_COLOR_SPACE", "YCbCr", + "IMAGE_STRUCTURE"); + GDALDataset::SetMetadataItem("COMPRESSION", "YCbCr JPEG", + "IMAGE_STRUCTURE"); + } + else + { + GDALDataset::SetMetadataItem("COMPRESSION", "JPEG", + "IMAGE_STRUCTURE"); + } + if (m_image->samplesPerPixel() != 1 && + m_image->samplesPerPixel() != 3 && + m_image->samplesPerPixel() != 4 && + m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Contiguous) + { + ReportError(CE_Failure, CPLE_NotSupported, + "Compression = JPEG not supported when samplesPerPixel " + "!= 1, 3 or 4 and planarConfiguration = Contiguous"); + return false; + } + + const auto psJPEGTablesTag = + m_image->tag(LIBERTIFF_NS::TagCode::JPEGTables); + if (psJPEGTablesTag && + psJPEGTablesTag->type == LIBERTIFF_NS::TagType::Undefined && + psJPEGTablesTag->count > 4 && + !psJPEGTablesTag->invalid_value_offset && + psJPEGTablesTag->count < 65536) + { + bool ok = true; + m_jpegTablesOri = + m_image->readTagAsVector(*psJPEGTablesTag, ok); + if (m_jpegTablesOri.size() >= 4 && m_jpegTablesOri[0] == 0xff && + m_jpegTablesOri[1] == 0xd8 && + m_jpegTablesOri[m_jpegTablesOri.size() - 2] == 0xff && + m_jpegTablesOri.back() == 0xd9) + { + m_jpegTables.insert( + m_jpegTables.end(), m_jpegTablesOri.data() + 2, + m_jpegTablesOri.data() + m_jpegTablesOri.size() - 2); + } + } + + if (m_image->samplesPerPixel() == 4 && + m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Contiguous) + { + const GByte abyAdobeAPP14RGB[] = { + 0xFF, 0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, + 0x65, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00}; + m_jpegTables.insert(m_jpegTables.end(), abyAdobeAPP14RGB, + abyAdobeAPP14RGB + sizeof(abyAdobeAPP14RGB)); + } + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::WEBP) + { + if (!GDALGetDriverByName("WEBP")) + { + ReportError( + CE_Failure, CPLE_NotSupported, + "Compression = WEBP not supported because WEBP driver missing"); + return false; + } + GDALDataset::SetMetadataItem("COMPRESSION", "WEBP", "IMAGE_STRUCTURE"); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::JXL || + m_image->compression() == LIBERTIFF_NS::Compression::JXL_DNG_1_7) + { + if (!GDALGetDriverByName("JPEGXL")) + { + ReportError( + CE_Failure, CPLE_NotSupported, + "Compression = JXL not supported because JXL driver missing"); + return false; + } + GDALDataset::SetMetadataItem("COMPRESSION", "JXL", "IMAGE_STRUCTURE"); + } + else if (m_image->compression() == LIBERTIFF_NS::Compression::LERC) + { +#ifndef LERC_SUPPORT + ReportError(CE_Failure, CPLE_NotSupported, + "Compression = LERC not supported because GDAL " + "has not been built against liblerc"); + return false; +#else + const auto *psLercParametersTag = + m_image->tag(LIBERTIFF_NS::TagCode::LERCParameters); + if (psLercParametersTag && + psLercParametersTag->type == LIBERTIFF_NS::TagType::Long && + psLercParametersTag->count == 2) + { + bool ok = true; + const auto lercParameters = + m_image->readTagAsVector(*psLercParametersTag, ok); + if (!ok || lercParameters.size() != 2) + { + ReportError(CE_Failure, CPLE_NotSupported, + "Tag LERCParameters is invalid"); + return false; + } + m_lercVersion = lercParameters[0]; + m_lercAdditionalCompression = lercParameters[1]; +#ifndef ZSTD_SUPPORT + if (m_lercAdditionalCompression == LERC_ADD_COMPRESSION_ZSTD) + { + ReportError( + CE_Failure, CPLE_NotSupported, + "Compression = LERC_ZSTD not supported because GDAL " + "has not been built against libzstd"); + return false; + } +#endif + } + + GDALDataset::SetMetadataItem( + "COMPRESSION", + m_lercAdditionalCompression == LERC_ADD_COMPRESSION_DEFLATE + ? "LERC_DEFLATE" + : m_lercAdditionalCompression == LERC_ADD_COMPRESSION_ZSTD + ? "LERC_ZSTD" + : "LERC", + "IMAGE_STRUCTURE"); + + if (m_lercVersion == LERC_VERSION_2_4) + { + GDALDataset::SetMetadataItem("LERC_VERSION", "2.4", + "IMAGE_STRUCTURE"); + } +#endif + } + else if (m_image->compression() != LIBERTIFF_NS::Compression::None) + { + CPLDebug("LIBERTIFF", "Compression = %s unhandled", + LIBERTIFF_NS::compressionName(m_image->compression())); + return false; + } + + return true; +} + +/************************************************************************/ +/* Open() */ +/************************************************************************/ + +bool LIBERTIFFDataset::Open(std::unique_ptr image) +{ + m_image = std::move(image); + + // Basic sanity checks + if (m_image->width() == 0 || + m_image->width() > static_cast(INT_MAX) || + m_image->height() == 0 || + m_image->height() > static_cast(INT_MAX) || + m_image->samplesPerPixel() == 0 || + m_image->samplesPerPixel() > static_cast(INT_MAX)) + { + CPLDebug("LIBERTIFF", "Invalid width, height, or samplesPerPixel"); + return false; + } + + nRasterXSize = static_cast(m_image->width()); + nRasterYSize = static_cast(m_image->height()); + const int l_nBands = static_cast(m_image->samplesPerPixel()); + if (!GDALCheckBandCount(l_nBands, false)) + return false; + + if (!ProcessCompressionMethod()) + return false; + + // Compute block size + int nBlockXSize; + int nBlockYSize; + if (m_image->isTiled()) + { + if (m_image->tileWidth() == 0 || + m_image->tileWidth() > static_cast(INT_MAX) || + m_image->tileHeight() == 0 || + m_image->tileHeight() > static_cast(INT_MAX)) + { + CPLDebug("LIBERTIFF", "Invalid tileWidth or tileHeight"); + return false; + } + nBlockXSize = static_cast(m_image->tileWidth()); + nBlockYSize = static_cast(m_image->tileHeight()); + } + else + { + if (m_image->rowsPerStripSanitized() == 0) + { + CPLDebug("LIBERTIFF", "Invalid rowsPerStrip"); + return false; + } + nBlockXSize = nRasterXSize; + nBlockYSize = static_cast(m_image->rowsPerStripSanitized()); + } + + const GDALDataType eDT = ComputeGDALDataType(); + + // Deal with Predictor tag + if (m_image->predictor() == 2) + { + GDALDataset::SetMetadataItem("PREDICTOR", "2", "IMAGE_STRUCTURE"); + } + else if (m_image->predictor() == 3) + { + if (eDT != GDT_Float32 && eDT != GDT_Float64) + { + CPLDebug("LIBERTIFF", "Unhandled predictor=3 with non-float data"); + return false; + } + GDALDataset::SetMetadataItem("PREDICTOR", "3", "IMAGE_STRUCTURE"); + } + else if (m_image->predictor() > 3) + { + CPLDebug("LIBERTIFF", "Predictor = %u unhandled", m_image->predictor()); + return false; + } + + // Deal with PlanarConfiguration tag + if (m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate || + m_image->samplesPerPixel() == 1) + GDALDataset::SetMetadataItem("INTERLEAVE", "BAND", "IMAGE_STRUCTURE"); + else + GDALDataset::SetMetadataItem("INTERLEAVE", "PIXEL", "IMAGE_STRUCTURE"); + + const int nNativeDTSize = GDALGetDataTypeSizeBytes(eDT); + const bool bSeparate = m_image->planarConfiguration() == + LIBERTIFF_NS::PlanarConfiguration::Separate; + // Sanity check that a strile can its on SIZE_MAX, to avoid further + // issues in ReadBlock() + if (static_cast(nNativeDTSize) * (bSeparate ? 1 : l_nBands) > + std::numeric_limits::max() / + (static_cast(nBlockXSize) * nBlockYSize)) + { + CPLDebug("LIBERTIFF", "Too large block"); + return false; + } + + // Process GDAL_NODATA tag + bool bHasNoData = false; + double dfNoData = 0; + const auto *tagNoData = m_image->tag(LIBERTIFF_NS::TagCode::GDAL_NODATA); + if (tagNoData && tagNoData->type == LIBERTIFF_NS::TagType::ASCII && + !(tagNoData->count > 4 && tagNoData->invalid_value_offset) && + tagNoData->count < 256) + { + bool ok = true; + const std::string noData = m_image->readTagAsString(*tagNoData, ok); + if (ok && !noData.empty()) + { + bHasNoData = true; + dfNoData = CPLAtof(noData.c_str()); + } + } + + // Process ExtraSamples tag + int nRegularChannels = 0; + if (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::MinIsBlack) + { + nRegularChannels = 1; + } + else if (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::RGB) + { + nRegularChannels = 3; + } + const auto *psExtraSamplesTag = + m_image->tag(LIBERTIFF_NS::TagCode::ExtraSamples); + if (nRegularChannels > 0 && l_nBands > nRegularChannels && + psExtraSamplesTag && + psExtraSamplesTag->type == LIBERTIFF_NS::TagType::Short && + psExtraSamplesTag->count == + static_cast(l_nBands - nRegularChannels)) + { + bool ok = true; + m_extraSamples = + m_image->readTagAsVector(*psExtraSamplesTag, ok); + } + + // Preload TileOffsets and TileByteCounts if not too big + if (m_image->isTiled()) + { + const auto *psTileOffsets = + m_image->tag(LIBERTIFF_NS::TagCode::TileOffsets); + const auto *psTileByteCounts = + m_image->tag(LIBERTIFF_NS::TagCode::TileByteCounts); + if (psTileOffsets && + (psTileOffsets->type == LIBERTIFF_NS::TagType::Long || + psTileOffsets->type == LIBERTIFF_NS::TagType::Long8) && + !psTileOffsets->invalid_value_offset && + psTileOffsets->count <= 4096 && psTileByteCounts && + psTileByteCounts->type == LIBERTIFF_NS::TagType::Long && + !psTileByteCounts->invalid_value_offset && + psTileByteCounts->count <= 4096) + { + bool ok = true; + if (psTileOffsets->type == LIBERTIFF_NS::TagType::Long) + m_tileOffsets = + m_image->readTagAsVector(*psTileOffsets, ok); + else + m_tileOffsets64 = + m_image->readTagAsVector(*psTileOffsets, ok); + m_tileByteCounts = + m_image->readTagAsVector(*psTileByteCounts, ok); + if (!ok) + { + m_tileOffsets.clear(); + m_tileOffsets64.clear(); + m_tileByteCounts.clear(); + } + } + } + + // Create raster bands + for (int i = 0; i < l_nBands; ++i) + { + auto poBand = std::make_unique(this, i + 1, eDT, + nBlockXSize, nBlockYSize); + poBand->m_bHasNoData = bHasNoData; + poBand->m_dfNoData = dfNoData; + if (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::MinIsBlack) + { + if (i == 0) + poBand->m_eColorInterp = GCI_GrayIndex; + } + else if (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::RGB || + (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::YCbCr && + m_image->samplesPerPixel() == 3)) + { + if (i < 3) + poBand->m_eColorInterp = + static_cast(GCI_RedBand + i); + } + if (i >= nRegularChannels && !m_extraSamples.empty()) + { + if (m_extraSamples[i - nRegularChannels] == + LIBERTIFF_NS::ExtraSamples::UnAssociatedAlpha) + { + poBand->m_eColorInterp = GCI_AlphaBand; + if (!m_poAlphaBand) + m_poAlphaBand = poBand.get(); + } + else if (m_extraSamples[i - nRegularChannels] == + LIBERTIFF_NS::ExtraSamples::AssociatedAlpha) + { + poBand->m_eColorInterp = GCI_AlphaBand; + poBand->GDALRasterBand::SetMetadataItem( + "ALPHA", "PREMULTIPLIED", "IMAGE_STRUCTURE"); + if (!m_poAlphaBand) + m_poAlphaBand = poBand.get(); + } + } + + if (m_image->bitsPerSample() != 8 && m_image->bitsPerSample() != 16 && + m_image->bitsPerSample() != 32 && m_image->bitsPerSample() != 64 && + m_image->bitsPerSample() != 128) + { + poBand->GDALRasterBand::SetMetadataItem( + "NBITS", CPLSPrintf("%u", m_image->bitsPerSample()), + "IMAGE_STRUCTURE"); + } + + if (l_nBands == 1 && eDT == GDT_Byte) + { + poBand->ReadColorMap(); + } + + if (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::MinIsWhite) + { + GDALDataset::SetMetadataItem("MINISWHITE", "YES", + "IMAGE_STRUCTURE"); + } + + if (m_image->bitsPerSample() == 1 && !poBand->m_poColorTable) + { + poBand->m_poColorTable = std::make_unique(); + const GDALColorEntry oEntryBlack = {0, 0, 0, 255}; + const GDALColorEntry oEntryWhite = {255, 255, 255, 255}; + if (m_image->photometricInterpretation() == + LIBERTIFF_NS::PhotometricInterpretation::MinIsWhite) + { + poBand->m_poColorTable->SetColorEntry(0, &oEntryWhite); + poBand->m_poColorTable->SetColorEntry(1, &oEntryBlack); + } + else + { + poBand->m_poColorTable->SetColorEntry(0, &oEntryBlack); + poBand->m_poColorTable->SetColorEntry(1, &oEntryWhite); + } + poBand->m_eColorInterp = GCI_PaletteIndex; + } + + SetBand(i + 1, std::move(poBand)); + } + + nOpenFlags = GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE; + + return true; +} + +/************************************************************************/ +/* Open() */ +/************************************************************************/ + +bool LIBERTIFFDataset::Open(GDALOpenInfo *poOpenInfo) +{ + SetDescription(poOpenInfo->pszFilename); + + int iSelectedSubDS = -1; + if (STARTS_WITH_CI(poOpenInfo->pszFilename, "GTIFF_DIR:")) + { + iSelectedSubDS = atoi(poOpenInfo->pszFilename + strlen("GTIFF_DIR:")); + if (iSelectedSubDS <= 0) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid subdataset syntax"); + return false; + } + const char *pszNextColon = + strchr(poOpenInfo->pszFilename + strlen("GTIFF_DIR:"), ':'); + if (!pszNextColon) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid subdataset syntax"); + return false; + } + m_poFile.reset(VSIFOpenL(pszNextColon + 1, "rb")); + if (!m_poFile) + { + CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", + pszNextColon + 1); + return false; + } + m_fileReader = + std::make_shared(m_poFile.get()); + } + else + { + m_fileReader = + std::make_shared(poOpenInfo->fpL); + } + + auto mainImage = LIBERTIFF_NS::open(m_fileReader); + if (!mainImage) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot open TIFF image"); + return false; + } + + if (mainImage->subFileType() != LIBERTIFF_NS::SubFileTypeFlags::Page && + mainImage->subFileType() != 0) + { + CPLDebug("LIBERTIFF", "Invalid subFileType value for first image"); + return false; + } + + // Check structural metadata (for COG) + const int nOffsetOfStructuralMetadata = + poOpenInfo->nHeaderBytes && ((poOpenInfo->pabyHeader[2] == 0x2B || + poOpenInfo->pabyHeader[3] == 0x2B)) + ? 16 + : 8; + if (poOpenInfo->nHeaderBytes > + nOffsetOfStructuralMetadata + + static_cast(strlen("GDAL_STRUCTURAL_METADATA_SIZE=")) && + memcmp(poOpenInfo->pabyHeader + nOffsetOfStructuralMetadata, + "GDAL_STRUCTURAL_METADATA_SIZE=", + strlen("GDAL_STRUCTURAL_METADATA_SIZE=")) == 0) + { + const char *pszStructuralMD = reinterpret_cast( + poOpenInfo->pabyHeader + nOffsetOfStructuralMetadata); + const bool bLayoutIFDSBeforeData = + strstr(pszStructuralMD, "LAYOUT=IFDS_BEFORE_DATA") != nullptr; + const bool bBlockOrderRowMajor = + strstr(pszStructuralMD, "BLOCK_ORDER=ROW_MAJOR") != nullptr; + const bool bLeaderSizeAsUInt4 = + strstr(pszStructuralMD, "BLOCK_LEADER=SIZE_AS_UINT4") != nullptr; + const bool bTrailerRepeatedLast4BytesRepeated = + strstr(pszStructuralMD, "BLOCK_TRAILER=LAST_4_BYTES_REPEATED") != + nullptr; + const bool bKnownIncompatibleEdition = + strstr(pszStructuralMD, "KNOWN_INCOMPATIBLE_EDITION=YES") != + nullptr; + if (bKnownIncompatibleEdition) + { + ReportError(CE_Warning, CPLE_AppDefined, + "This file used to have optimizations in its layout, " + "but those have been, at least partly, invalidated by " + "later changes"); + } + else if (bLayoutIFDSBeforeData && bBlockOrderRowMajor && + bLeaderSizeAsUInt4 && bTrailerRepeatedLast4BytesRepeated) + { + GDALDataset::SetMetadataItem("LAYOUT", "COG", "IMAGE_STRUCTURE"); + } + } + + if (!Open(std::move(mainImage))) + return false; + + // Iterate over overviews + LIBERTIFFDataset *poLastNonMaskDS = this; + auto imageNext = m_image->next(); + if (imageNext && + (m_image->subFileType() == 0 || + m_image->subFileType() == LIBERTIFF_NS::SubFileTypeFlags::Page) && + (imageNext->subFileType() == 0 || + imageNext->subFileType() == LIBERTIFF_NS::SubFileTypeFlags::Page)) + { + int iSubDS = 1; + CPLStringList aosList; + auto curImage = std::move(m_image); + do + { + if (iSelectedSubDS > 0 && iSubDS == iSelectedSubDS) + { + m_image = std::move(curImage); + break; + } + if (iSelectedSubDS < 0) + { + aosList.AddNameValue( + CPLSPrintf("SUBDATASET_%d_NAME", iSubDS), + CPLSPrintf("GTIFF_DIR:%d:%s", iSubDS, GetDescription())); + aosList.AddNameValue(CPLSPrintf("SUBDATASET_%d_DESC", iSubDS), + CPLSPrintf("Page %d (%uP x %uL x %uB)", + iSubDS, curImage->width(), + curImage->height(), + curImage->samplesPerPixel())); + } + ++iSubDS; + if (iSubDS == 65536) + { + ReportError(CE_Warning, CPLE_AppDefined, + "Stopping IFD scanning at 65536th one"); + break; + } + curImage = curImage->next(); + } while (curImage); + if (iSelectedSubDS < 0) + { + for (int i = 0; i < nBands; ++i) + delete papoBands[i]; + CPLFree(papoBands); + papoBands = nullptr; + nRasterXSize = 0; + nRasterYSize = 0; + GDALDataset::SetMetadata(aosList.List(), "SUBDATASETS"); + return true; + } + else if (!m_image) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot open %dth image", + iSelectedSubDS); + return false; + } + } + else if (iSelectedSubDS < 0) + { + auto curImage = std::move(imageNext); + int iters = 0; + while (curImage) + { + auto nextImage = curImage->next(); + if (curImage->subFileType() == + LIBERTIFF_NS::SubFileTypeFlags::ReducedImage) + { + // Overview IFD + auto poOvrDS = std::make_unique(); + if (poOvrDS->Open(std::move(curImage)) && + poOvrDS->GetRasterCount() == nBands && + poOvrDS->GetRasterXSize() <= nRasterXSize && + poOvrDS->GetRasterYSize() <= nRasterYSize && + poOvrDS->GetRasterBand(1)->GetRasterDataType() == + GetRasterBand(1)->GetRasterDataType()) + { + m_apoOvrDSOwned.push_back(std::move(poOvrDS)); + auto poOvrDSRaw = m_apoOvrDSOwned.back().get(); + m_apoOvrDS.push_back(poOvrDSRaw); + poLastNonMaskDS = poOvrDSRaw; + } + } + else if ((curImage->subFileType() & + LIBERTIFF_NS::SubFileTypeFlags::Mask) != 0) + { + // Mask IFD + if (!poLastNonMaskDS->m_poMaskDS) + { + auto poMaskDS = std::make_unique(); + if (poMaskDS->Open(std::move(curImage)) && + poMaskDS->GetRasterCount() == 1 && + poMaskDS->GetRasterXSize() == + poLastNonMaskDS->nRasterXSize && + poMaskDS->GetRasterYSize() == + poLastNonMaskDS->nRasterYSize && + poMaskDS->GetRasterBand(1)->GetRasterDataType() == + GDT_Byte) + { + poMaskDS->m_bExpand1To255 = true; + poLastNonMaskDS->m_poMaskDS = std::move(poMaskDS); + if (poLastNonMaskDS != this && m_poMaskDS) + { + // Also register the mask as the overview of the main + // mask + m_poMaskDS->m_apoOvrDS.push_back( + poLastNonMaskDS->m_poMaskDS.get()); + } + } + } + } + else + { + CPLDebug("LIBERTIFF", + "Unhandled subFileType value for auxiliary image"); + return false; + } + curImage = std::move(nextImage); + + ++iters; + if (iters == 64) + { + ReportError(CE_Warning, CPLE_AppDefined, + "Stopping IFD scanning at 64th one"); + break; + } + } + } + + static const struct + { + LIBERTIFF_NS::TagCodeType code; + const char *mditem; + } strTags[] = { + {LIBERTIFF_NS::TagCode::DocumentName, "TIFFTAG_DOCUMENTNAME"}, + {LIBERTIFF_NS::TagCode::ImageDescription, "TIFFTAG_IMAGEDESCRIPTION"}, + {LIBERTIFF_NS::TagCode::Software, "TIFFTAG_SOFTWARE"}, + {LIBERTIFF_NS::TagCode::DateTime, "TIFFTAG_DATETIME"}, + }; + + for (const auto &strTag : strTags) + { + const auto *tag = m_image->tag(strTag.code); + constexpr size_t ARBITRARY_MAX_SIZE = 65536; + if (tag && tag->type == LIBERTIFF_NS::TagType::ASCII && + !(tag->count > 4 && tag->invalid_value_offset) && + tag->count < ARBITRARY_MAX_SIZE) + { + bool ok = true; + const std::string str = m_image->readTagAsString(*tag, ok); + if (ok) + { + GDALDataset::SetMetadataItem(strTag.mditem, str.c_str()); + } + } + } + + ReadSRS(); + ReadGeoTransform(); + + const auto *psGDALMetadataTag = + m_image->tag(LIBERTIFF_NS::TagCode::GDAL_METADATA); + constexpr size_t ARBITRARY_MAX_SIZE_GDAL_METADATA = 10 * 1024 * 1024; + if (psGDALMetadataTag && + psGDALMetadataTag->type == LIBERTIFF_NS::TagType::ASCII && + !(psGDALMetadataTag->count > 4 && + psGDALMetadataTag->invalid_value_offset) && + psGDALMetadataTag->count < ARBITRARY_MAX_SIZE_GDAL_METADATA) + { + bool ok = true; + const std::string str = + m_image->readTagAsString(*psGDALMetadataTag, ok); + if (ok) + { + auto oRoot = CPLXMLTreeCloser(CPLParseXMLString(str.c_str())); + if (oRoot.get()) + { + const CPLXMLNode *psItem = + oRoot.get() ? CPLGetXMLNode(oRoot.get(), "=GDALMetadata") + : nullptr; + if (psItem) + psItem = psItem->psChild; + for (; psItem != nullptr; psItem = psItem->psNext) + { + if (psItem->eType != CXT_Element || + !EQUAL(psItem->pszValue, "Item")) + continue; + + const char *pszKey = + CPLGetXMLValue(psItem, "name", nullptr); + const char *pszValue = + CPLGetXMLValue(psItem, nullptr, nullptr); + int nBand = atoi(CPLGetXMLValue(psItem, "sample", "-1")); + if (nBand < -1 || nBand > 65535) + continue; + nBand++; + const char *pszRole = CPLGetXMLValue(psItem, "role", ""); + const char *pszDomain = + CPLGetXMLValue(psItem, "domain", ""); + + if (pszKey == nullptr || pszValue == nullptr) + continue; + if (EQUAL(pszDomain, "IMAGE_STRUCTURE")) + { + if (m_image->compression() == + LIBERTIFF_NS::Compression::WEBP && + EQUAL(pszKey, "COMPRESSION_REVERSIBILITY")) + { + // go on + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::WEBP && + EQUAL(pszKey, "WEBP_LEVEL")) + { + const int nLevel = atoi(pszValue); + if (nLevel >= 1 && nLevel <= 100) + { + GDALDataset::SetMetadataItem( + "COMPRESSION_REVERSIBILITY", "LOSSY", + "IMAGE_STRUCTURE"); + } + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::LERC && + EQUAL(pszKey, "MAX_Z_ERROR")) + { + // go on + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::LERC && + EQUAL(pszKey, "MAX_Z_ERROR_OVERVIEW")) + { + // go on + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::JXL && + EQUAL(pszKey, "COMPRESSION_REVERSIBILITY")) + { + // go on + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::JXL && + EQUAL(pszKey, "JXL_DISTANCE")) + { + const double dfVal = CPLAtof(pszValue); + if (dfVal > 0 && dfVal <= 15) + { + GDALDataset::SetMetadataItem( + "COMPRESSION_REVERSIBILITY", "LOSSY", + "IMAGE_STRUCTURE"); + } + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::JXL && + EQUAL(pszKey, "JXL_ALPHA_DISTANCE")) + { + const double dfVal = CPLAtof(pszValue); + if (dfVal > 0 && dfVal <= 15) + { + GDALDataset::SetMetadataItem( + "COMPRESSION_REVERSIBILITY", "LOSSY", + "IMAGE_STRUCTURE"); + } + } + else if (m_image->compression() == + LIBERTIFF_NS::Compression::JXL && + EQUAL(pszKey, "JXL_EFFORT")) + { + // go on + } + else + { + continue; + } + } + + bool bIsXML = false; + + if (STARTS_WITH_CI(pszDomain, "xml:")) + bIsXML = TRUE; + + // Note: this un-escaping should not normally be done, as the + // deserialization of the tree from XML also does it, so we end up + // width double XML escaping, but keep it for backward + // compatibility. + char *pszUnescapedValue = + CPLUnescapeString(pszValue, nullptr, CPLES_XML); + if (nBand == 0) + { + if (bIsXML) + { + char *apszMD[2] = {pszUnescapedValue, nullptr}; + GDALDataset::SetMetadata(apszMD, pszDomain); + } + else + { + GDALDataset::SetMetadataItem( + pszKey, pszUnescapedValue, pszDomain); + } + } + else + { + auto poBand = cpl::down_cast( + GetRasterBand(nBand)); + if (poBand != nullptr) + { + if (EQUAL(pszRole, "scale")) + { + poBand->m_bHaveOffsetScale = true; + poBand->m_dfScale = CPLAtofM(pszUnescapedValue); + } + else if (EQUAL(pszRole, "offset")) + { + poBand->m_bHaveOffsetScale = true; + poBand->m_dfOffset = + CPLAtofM(pszUnescapedValue); + } + else if (EQUAL(pszRole, "unittype")) + { + poBand->m_osUnitType = pszUnescapedValue; + } + else if (EQUAL(pszRole, "description")) + { + poBand->m_osDescription = pszUnescapedValue; + } + else if (EQUAL(pszRole, "colorinterp")) + { + if (EQUAL(pszUnescapedValue, "undefined")) + poBand->m_eColorInterp = GCI_Undefined; + else + { + poBand->m_eColorInterp = + GDALGetColorInterpretationByName( + pszUnescapedValue); + if (poBand->m_eColorInterp == GCI_Undefined) + { + poBand->GDALRasterBand::SetMetadataItem( + "COLOR_INTERPRETATION", + pszUnescapedValue); + } + } + } + else + { + if (bIsXML) + { + char *apszMD[2] = {pszUnescapedValue, + nullptr}; + poBand->GDALRasterBand::SetMetadata( + apszMD, pszDomain); + } + else + { + poBand->GDALRasterBand::SetMetadataItem( + pszKey, pszUnescapedValue, pszDomain); + } + } + } + } + CPLFree(pszUnescapedValue); + } + } + } + } + + if ((m_image->compression() == LIBERTIFF_NS::Compression::WEBP || + m_image->compression() == LIBERTIFF_NS::Compression::JXL || + m_image->compression() == LIBERTIFF_NS::Compression::JXL_DNG_1_7) && + GetMetadataItem("COMPRESSION_REVERSIBILITY", "IMAGE_STRUCTURE") == + nullptr) + { + const char *pszDriverName = + m_image->compression() == LIBERTIFF_NS::Compression::WEBP + ? "WEBP" + : "JPEGXL"; + auto poTileDriver = GDALGetDriverByName(pszDriverName); + if (poTileDriver) + { + bool ok = true; + const uint64_t offset = m_image->strileOffset(0, ok); + const uint64_t bytecount = m_image->strileByteCount(0, ok); + if (ok && bytecount > 0) + { + const std::string osSubfile( + CPLSPrintf("/vsisubfile/" CPL_FRMT_GUIB "_%d,%s", + static_cast(offset), + static_cast(std::min( + static_cast(1024), bytecount)), + GetDescription())); + const char *const apszDrivers[] = {pszDriverName, nullptr}; + auto poTileDataset = + std::unique_ptr(GDALDataset::Open( + osSubfile.c_str(), GDAL_OF_RASTER, apszDrivers)); + if (poTileDataset) + { + const char *pszReversibility = + poTileDataset->GetMetadataItem( + "COMPRESSION_REVERSIBILITY", "IMAGE_STRUCTURE"); + if (pszReversibility) + GDALDataset::SetMetadataItem( + "COMPRESSION_REVERSIBILITY", pszReversibility, + "IMAGE_STRUCTURE"); + } + } + } + } + + // Init mask bands + for (int i = 0; i < nBands; ++i) + { + cpl::down_cast(papoBands[i])->InitMaskBand(); + } + for (auto &poOvrDS : m_apoOvrDS) + { + for (int i = 0; i < nBands; ++i) + { + cpl::down_cast(poOvrDS->papoBands[i]) + ->InitMaskBand(); + } + } + + m_fileReader->setPReadAllowed(); + + if (poOpenInfo->fpL) + { + m_poFile.reset(poOpenInfo->fpL); + poOpenInfo->fpL = nullptr; + } + + return true; +} + +/************************************************************************/ +/* ReadSRS() */ +/************************************************************************/ + +// Simplified GeoTIFF SRS reader, assuming the SRS is encoded as a EPSG code +void LIBERTIFFDataset::ReadSRS() +{ + const auto psGeoKeysTag = + m_image->tag(LIBERTIFF_NS::TagCode::GeoTIFFGeoKeyDirectory); + constexpr int VALUES_PER_GEOKEY = 4; + if (psGeoKeysTag && psGeoKeysTag->type == LIBERTIFF_NS::TagType::Short && + !psGeoKeysTag->invalid_value_offset && + psGeoKeysTag->count >= VALUES_PER_GEOKEY && + (psGeoKeysTag->count % VALUES_PER_GEOKEY) == 0 && + // Sanity check + psGeoKeysTag->count < 1000) + { + bool ok = true; + const auto values = + m_image->readTagAsVector(*psGeoKeysTag, ok); + if (values.size() >= 4) + { + const uint16_t geokeysCount = values[3]; + constexpr uint16_t GEOTIFF_KEY_DIRECTORY_VERSION_V1 = 1; + constexpr uint16_t GEOTIFF_KEY_VERSION_MAJOR_V1 = 1; + if (values[0] == GEOTIFF_KEY_DIRECTORY_VERSION_V1 && + // GeoTIFF 1.x + values[1] == GEOTIFF_KEY_VERSION_MAJOR_V1 && + // No equality for autotest/gcore/data/ycbcr_with_mask.tif + geokeysCount <= psGeoKeysTag->count / VALUES_PER_GEOKEY - 1) + { + constexpr uint16_t GeoTIFFTypeShort = 0; + constexpr uint16_t GeoTIFFTypeDouble = + LIBERTIFF_NS::TagCode::GeoTIFFDoubleParams; + + constexpr uint16_t GTModelTypeGeoKey = 1024; + constexpr uint16_t ModelTypeProjected = 1; + constexpr uint16_t ModelTypeGeographic = 2; + + constexpr uint16_t GTRasterTypeGeoKey = 1025; + constexpr uint16_t RasterPixelIsArea = 1; + constexpr uint16_t RasterPixelIsPoint = 2; + + constexpr uint16_t GeodeticCRSGeoKey = 2048; + constexpr uint16_t ProjectedCRSGeoKey = 3072; + + constexpr uint16_t CoordinateEpochGeoKey = 5120; + + uint16_t nModelType = 0; + uint16_t nEPSGCode = 0; + double dfCoordEpoch = 0; + bool bHasCoordEpoch = false; + for (uint32_t i = 1; i <= geokeysCount; ++i) + { + const auto geokey = values[VALUES_PER_GEOKEY * i]; + const auto geokeyType = values[VALUES_PER_GEOKEY * i + 1]; + const auto geokeyCount = values[VALUES_PER_GEOKEY * i + 2]; + const auto geokeyValue = values[VALUES_PER_GEOKEY * i + 3]; + if (geokey == GTModelTypeGeoKey) + { + nModelType = geokeyValue; + } + else if (geokey == GeodeticCRSGeoKey && + nModelType == ModelTypeGeographic && + geokeyType == GeoTIFFTypeShort && + geokeyCount == 1 && geokeyValue > 0) + { + nEPSGCode = geokeyValue; + } + else if (geokey == ProjectedCRSGeoKey && + nModelType == ModelTypeProjected && + geokeyType == GeoTIFFTypeShort && + geokeyCount == 1 && geokeyValue > 0) + { + nEPSGCode = geokeyValue; + } + else if (geokey == GTRasterTypeGeoKey && + geokeyType == GeoTIFFTypeShort && geokeyCount == 1) + { + if (geokeyValue == RasterPixelIsArea) + { + GDALDataset::SetMetadataItem(GDALMD_AREA_OR_POINT, + GDALMD_AOP_AREA); + } + else if (geokeyValue == RasterPixelIsPoint) + { + GDALDataset::SetMetadataItem(GDALMD_AREA_OR_POINT, + GDALMD_AOP_POINT); + } + } + else if (geokey == CoordinateEpochGeoKey && + geokeyType == GeoTIFFTypeDouble && + geokeyCount == 1) + { + const auto psGeoDoubleParamsTag = m_image->tag( + LIBERTIFF_NS::TagCode::GeoTIFFDoubleParams); + if (psGeoDoubleParamsTag && + psGeoDoubleParamsTag->type == + LIBERTIFF_NS::TagType::Double && + psGeoDoubleParamsTag->count > geokeyValue) + { + ok = true; + const auto doubleValues = + m_image->readTagAsVector( + *psGeoDoubleParamsTag, ok); + if (ok && doubleValues.size() > geokeyValue) + { + bHasCoordEpoch = true; + dfCoordEpoch = doubleValues[geokeyValue]; + } + } + } + } + + if (nEPSGCode > 0 && nEPSGCode != 32767) + { + m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + m_oSRS.importFromEPSG(nEPSGCode); + if (bHasCoordEpoch) + m_oSRS.SetCoordinateEpoch(dfCoordEpoch); + return; + } + + const char *const apszAllowedDrivers[] = {"GTiff", nullptr}; + auto poTmpDS = std::unique_ptr(GDALDataset::Open( + GetDescription(), GDAL_OF_RASTER | GDAL_OF_INTERNAL, + apszAllowedDrivers, nullptr, nullptr)); + if (poTmpDS) + { + const OGRSpatialReference *poSRS = poTmpDS->GetSpatialRef(); + if (!poSRS) + poSRS = poTmpDS->GetGCPSpatialRef(); + if (poSRS) + m_oSRS = *poSRS; + } + } + } + } +} + +/************************************************************************/ +/* ReadGeoTransform() */ +/************************************************************************/ + +void LIBERTIFFDataset::ReadGeoTransform() +{ + // Number of values per GCP in the GeoTIFFTiePoints tag + constexpr int VALUES_PER_GCP = 6; + + constexpr int GCP_PIXEL = 0; + constexpr int GCP_LINE = 1; + // constexpr int GCP_DEPTH = 2; + constexpr int GCP_X = 3; + constexpr int GCP_Y = 4; + constexpr int GCP_Z = 5; + + const auto *psTagTiePoints = + m_image->tag(LIBERTIFF_NS::TagCode::GeoTIFFTiePoints); + const auto *psTagPixelScale = + m_image->tag(LIBERTIFF_NS::TagCode::GeoTIFFPixelScale); + const auto *psTagGeoTransMatrix = + m_image->tag(LIBERTIFF_NS::TagCode::GeoTIFFGeoTransMatrix); + if (psTagTiePoints && + psTagTiePoints->type == LIBERTIFF_NS::TagType::Double && + !psTagTiePoints->invalid_value_offset && + psTagTiePoints->count == VALUES_PER_GCP && psTagPixelScale && + psTagPixelScale->type == LIBERTIFF_NS::TagType::Double && + !psTagPixelScale->invalid_value_offset && psTagPixelScale->count == 3) + { + bool ok = true; + const auto tiepoints = + m_image->readTagAsVector(*psTagTiePoints, ok); + const auto pixelScale = + m_image->readTagAsVector(*psTagPixelScale, ok); + if (!ok) + return; + + m_geotransformValid = true; + m_geotransform[1] = pixelScale[GCP_PIXEL]; + m_geotransform[5] = -pixelScale[GCP_LINE]; + m_geotransform[0] = + tiepoints[GCP_X] - tiepoints[GCP_PIXEL] * m_geotransform[1]; + m_geotransform[3] = + tiepoints[GCP_Y] - tiepoints[GCP_LINE] * m_geotransform[5]; + } + else if (psTagGeoTransMatrix && + psTagGeoTransMatrix->type == LIBERTIFF_NS::TagType::Double && + !psTagGeoTransMatrix->invalid_value_offset && + psTagGeoTransMatrix->count == 16) + { + bool ok = true; + const auto matrix = + m_image->readTagAsVector(*psTagGeoTransMatrix, ok); + if (ok) + { + m_geotransformValid = true; + m_geotransform[0] = matrix[3]; + m_geotransform[1] = matrix[0]; + m_geotransform[2] = matrix[1]; + m_geotransform[3] = matrix[7]; + m_geotransform[4] = matrix[4]; + m_geotransform[5] = matrix[5]; + } + } + else if (psTagTiePoints && + psTagTiePoints->type == LIBERTIFF_NS::TagType::Double && + !psTagTiePoints->invalid_value_offset && + psTagTiePoints->count > VALUES_PER_GCP && + (psTagTiePoints->count % VALUES_PER_GCP) == 0 && + psTagTiePoints->count <= 10000 * VALUES_PER_GCP) + { + bool ok = true; + const auto tiepoints = + m_image->readTagAsVector(*psTagTiePoints, ok); + if (ok) + { + bool pixelIsPoint = false; + if (const char *pszAreaOrPoint = + GetMetadataItem(GDALMD_AREA_OR_POINT)) + { + pixelIsPoint = EQUAL(pszAreaOrPoint, GDALMD_AOP_POINT); + } + const int gcpCount = + static_cast(psTagTiePoints->count / VALUES_PER_GCP); + for (int iGCP = 0; iGCP < gcpCount; ++iGCP) + { + m_aoGCPs.emplace_back( + CPLSPrintf("%d", iGCP + 1), "", + /* pixel = */ tiepoints[iGCP * VALUES_PER_GCP + GCP_PIXEL], + /* line = */ tiepoints[iGCP * VALUES_PER_GCP + GCP_LINE], + /* X = */ tiepoints[iGCP * VALUES_PER_GCP + GCP_X], + /* Y = */ tiepoints[iGCP * VALUES_PER_GCP + GCP_Y], + /* Z = */ tiepoints[iGCP * VALUES_PER_GCP + GCP_Z]); + + if (pixelIsPoint) + { + m_aoGCPs.back().Pixel() += 0.5; + m_aoGCPs.back().Line() += 0.5; + } + } + } + } + + if (m_geotransformValid) + { + if (const char *pszAreaOrPoint = GetMetadataItem(GDALMD_AREA_OR_POINT)) + { + if (EQUAL(pszAreaOrPoint, GDALMD_AOP_POINT)) + { + m_geotransform[0] -= + (m_geotransform[1] * 0.5 + m_geotransform[2] * 0.5); + m_geotransform[3] -= + (m_geotransform[4] * 0.5 + m_geotransform[5] * 0.5); + } + } + } +} + +/************************************************************************/ +/* OpenStatic() */ +/************************************************************************/ + +/* static */ GDALDataset *LIBERTIFFDataset::OpenStatic(GDALOpenInfo *poOpenInfo) +{ + if (!Identify(poOpenInfo)) + return nullptr; + + auto poDS = std::make_unique(); + if (!poDS->Open(poOpenInfo)) + return nullptr; + return poDS.release(); +} + +/************************************************************************/ +/* GDALRegister_LIBERTIFF() */ +/************************************************************************/ + +void GDALRegister_LIBERTIFF() + +{ + if (GDALGetDriverByName("LIBERTIFF") != nullptr) + return; + + auto poDriver = std::make_unique(); + poDriver->SetDescription("LIBERTIFF"); + poDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES"); + poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, + "GeoTIFF (using LIBERTIFF library)"); + poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, + "drivers/raster/libertiff.html"); + poDriver->SetMetadataItem(GDAL_DMD_MIMETYPE, "image/tiff"); + poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "tif tiff"); + poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); + poDriver->SetMetadataItem(GDAL_DCAP_COORDINATE_EPOCH, "YES"); + + poDriver->pfnIdentify = LIBERTIFFDataset::Identify; + poDriver->pfnOpen = LIBERTIFFDataset::OpenStatic; + + if (CPLGetDecompressor("lzma")) + { + poDriver->SetMetadataItem("LZMA_SUPPORT", "YES", "LIBERTIFF"); + } +#ifdef ZSTD_SUPPORT + poDriver->SetMetadataItem("ZSTD_SUPPORT", "YES", "LIBERTIFF"); +#endif +#if defined(LERC_SUPPORT) + poDriver->SetMetadataItem("LERC_SUPPORT", "YES", "LIBERTIFF"); +#if defined(LERC_VERSION_MAJOR) + poDriver->SetMetadataItem("LERC_VERSION_MAJOR", + XSTRINGIFY(LERC_VERSION_MAJOR), "LERC"); + poDriver->SetMetadataItem("LERC_VERSION_MINOR", + XSTRINGIFY(LERC_VERSION_MINOR), "LERC"); + poDriver->SetMetadataItem("LERC_VERSION_PATCH", + XSTRINGIFY(LERC_VERSION_PATCH), "LERC"); +#endif +#endif + + GetGDALDriverManager()->RegisterDriver(poDriver.release()); +} diff --git a/frmts/libertiff/libtiff_codecs.h b/frmts/libertiff/libtiff_codecs.h new file mode 100644 index 000000000000..db4c881ddd7d --- /dev/null +++ b/frmts/libertiff/libtiff_codecs.h @@ -0,0 +1,127 @@ +/****************************************************************************** + * + * Project: GDAL Core + * Purpose: GeoTIFF thread safe reader using libertiff library. + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +// Use code from internal libtiff for LZW and PackBits codecs +#define TIFFInitLZW LIBERTIFF_TIFFInitLZW +#define TIFFInitPackBits LIBERTIFF_TIFFInitPackBits +#define TIFFInitLERC LIBERTIFF_TIFFInitLERC +#define _TIFFmallocExt LIBERTIFF_TIFFmallocExt +#define _TIFFreallocExt LIBERTIFF_TIFFreallocExt +#define _TIFFcallocExt LIBERTIFF_TIFFcallocExt +#define _TIFFfreeExt LIBERTIFF_TIFFfreeExt +#define _TIFFmemset LIBERTIFF_TIFFmemset +#define _TIFFmemcpy LIBERTIFF_TIFFmemcpy +#define TIFFPredictorInit LIBERTIFF_TIFFPredictorInit +#define TIFFPredictorCleanup LIBERTIFF_TIFFPredictorCleanup +#define _TIFFSetDefaultCompressionState LIBERTIFF_TIFFSetDefaultCompressionState +#define TIFFFlushData1 LIBERTIFF_TIFFFlushData1_dummy +#define TIFFWarningExtR LIBERTIFF_TIFFWarningExtR +#define TIFFErrorExtR LIBERTIFF_TIFFErrorExtR +#define TIFFTileRowSize LIBERTIFF_TIFFTileRowSize_dummy +#define TIFFScanlineSize LIBERTIFF_TIFFScanlineSize_dummy +#define TIFFSetField LIBERTIFF_TIFFSetField_dummy +#define _TIFFMergeFields LIBERTIFF_TIFFMergeFields_dummy +#define register +extern "C" +{ +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#endif + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif + +#define LZW_READ_ONLY +#include "tif_lzw.c" + +#define PACKBITS_READ_ONLY +#include "tif_packbits.c" + +#ifdef LERC_SUPPORT +#define LERC_READ_ONLY +#include "tif_lerc.c" +#endif + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + + void *LIBERTIFF_TIFFmallocExt(TIFF *, tmsize_t s) + { + return malloc(s); + } + + void *LIBERTIFF_TIFFreallocExt(TIFF *, void *p, tmsize_t s) + { + return realloc(p, s); + } + + void *LIBERTIFF_TIFFcallocExt(TIFF *, tmsize_t nmemb, tmsize_t siz) + { + return calloc(nmemb, siz); + } + + void LIBERTIFF_TIFFfreeExt(TIFF *, void *ptr) + { + free(ptr); + } + + void LIBERTIFF_TIFFmemset(void *ptr, int v, tmsize_t s) + { + memset(ptr, v, s); + } + + void LIBERTIFF_TIFFmemcpy(void *d, const void *s, tmsize_t c) + { + memcpy(d, s, c); + } + + void LIBERTIFF_TIFFSetDefaultCompressionState(TIFF *) + { + } + + int LIBERTIFF_TIFFSetField_dummy(TIFF *, uint32_t, ...) + { + return 0; + } + + int LIBERTIFF_TIFFMergeFields_dummy(TIFF *, const TIFFField[], uint32_t) + { + return 1; + } + + int LIBERTIFF_TIFFPredictorInit(TIFF *) + { + return 0; + } + + int LIBERTIFF_TIFFPredictorCleanup(TIFF *) + { + return 0; + } + + void LIBERTIFF_TIFFWarningExtR(TIFF *, const char *, const char *, ...) + { + } + + void LIBERTIFF_TIFFErrorExtR(TIFF *, const char *, const char *, ...) + { + } +} +#undef isTiled diff --git a/gcore/gdal_frmts.h b/gcore/gdal_frmts.h index 610b12ddb07a..e957afc14719 100644 --- a/gcore/gdal_frmts.h +++ b/gcore/gdal_frmts.h @@ -18,6 +18,7 @@ CPL_C_START void CPL_DLL GDALRegister_GTiff(void); +void CPL_DLL GDALRegister_LIBERTIFF(void); void CPL_DLL GDALRegister_GXF(void); void CPL_DLL GDALRegister_HFA(void); void CPL_DLL GDALRegister_AAIGrid(void); diff --git a/gcore/gdal_priv.h b/gcore/gdal_priv.h index f5501e1a91d6..7418fdb3d074 100644 --- a/gcore/gdal_priv.h +++ b/gcore/gdal_priv.h @@ -1574,6 +1574,12 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject m_poBandRef = bOwned ? nullptr : poBand; } + void reset(std::unique_ptr poBand) + { + m_poBandOwned = std::move(poBand); + m_poBandRef = nullptr; + } + const GDALRasterBand *get() const { return static_cast(*this); @@ -1968,6 +1974,12 @@ class CPL_DLL GDALAllValidMaskBand : public GDALRasterBand protected: CPLErr IReadBlock(int, int, void *) override; + CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, GSpacing nPixelSpace, + GSpacing nLineSpace, + GDALRasterIOExtraArg *psExtraArg) override; + CPL_DISALLOW_COPY_ASSIGN(GDALAllValidMaskBand) public: diff --git a/gcore/gdalallvalidmaskband.cpp b/gcore/gdalallvalidmaskband.cpp index cc44b5045de7..b34bee6a18c4 100644 --- a/gcore/gdalallvalidmaskband.cpp +++ b/gcore/gdalallvalidmaskband.cpp @@ -55,6 +55,33 @@ CPLErr GDALAllValidMaskBand::IReadBlock(int /* nXBlockOff */, return CE_None; } +/************************************************************************/ +/* IRasterIO() */ +/************************************************************************/ + +CPLErr GDALAllValidMaskBand::IRasterIO(GDALRWFlag eRWFlag, int, int, int, int, + void *pData, int nBufXSize, + int nBufYSize, GDALDataType eBufType, + GSpacing nPixelSpace, + GSpacing nLineSpace, + GDALRasterIOExtraArg *) +{ + if (eRWFlag != GF_Read) + { + return CE_Failure; + } + + GByte *pabyData = static_cast(pData); + GByte byVal = 255; + for (int iY = 0; iY < nBufYSize; ++iY) + { + GDALCopyWords64(&byVal, GDT_Byte, 0, pabyData + iY * nLineSpace, + eBufType, static_cast(nPixelSpace), nBufXSize); + } + + return CE_None; +} + /************************************************************************/ /* GetMaskBand() */ /************************************************************************/ diff --git a/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py b/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py index 52e087e55128..22ccaa6d2179 100644 --- a/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py +++ b/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py @@ -162,7 +162,7 @@ def validate(ds, check_tiled=True, full_check=False): raise ValidateCloudOptimizedGeoTIFFException( "Invalid file : %s" % gdal.GetLastErrorMsg() ) - if ds.GetDriver().ShortName != "GTiff": + if ds.GetDriver().ShortName not in ("GTiff", "LIBERTIFF"): raise ValidateCloudOptimizedGeoTIFFException("The file is not a GeoTIFF") details = {} diff --git a/third_party/libertiff/libertiff.hpp b/third_party/libertiff/libertiff.hpp index 5275b6115d2b..e50a8f08c1d0 100644 --- a/third_party/libertiff/libertiff.hpp +++ b/third_party/libertiff/libertiff.hpp @@ -14,6 +14,7 @@ #include // std::endian #endif +#include #include #include #include @@ -378,8 +379,8 @@ typedef uint16_t TagCodeType; /** TIFF tag codes */ namespace TagCode { -constexpr TagCodeType NewSubfileType = 254; -constexpr TagCodeType SubfileType = 255; +constexpr TagCodeType SubFileType = 254; +constexpr TagCodeType OldSubFileType = 255; // Base line and extended TIFF tags constexpr TagCodeType ImageWidth = 256; @@ -387,23 +388,33 @@ constexpr TagCodeType ImageLength = 257; constexpr TagCodeType BitsPerSample = 258; constexpr TagCodeType Compression = 259; constexpr TagCodeType PhotometricInterpretation = 262; +constexpr TagCodeType DocumentName = 269; constexpr TagCodeType ImageDescription = 270; constexpr TagCodeType StripOffsets = 273; constexpr TagCodeType SamplesPerPixel = 277; constexpr TagCodeType RowsPerStrip = 278; constexpr TagCodeType StripByteCounts = 279; constexpr TagCodeType PlanarConfiguration = 284; +constexpr TagCodeType Software = 305; +constexpr TagCodeType DateTime = 306; constexpr TagCodeType Predictor = 317; +constexpr TagCodeType ColorMap = 320; constexpr TagCodeType TileWidth = 322; constexpr TagCodeType TileLength = 323; constexpr TagCodeType TileOffsets = 324; constexpr TagCodeType TileByteCounts = 325; +constexpr TagCodeType ExtraSamples = 338; constexpr TagCodeType SampleFormat = 339; +constexpr TagCodeType JPEGTables = 347; + +constexpr TagCodeType Copyright = 33432; // GeoTIFF tags constexpr TagCodeType GeoTIFFPixelScale = 33550; constexpr TagCodeType GeoTIFFTiePoints = 33922; +constexpr TagCodeType GeoTIFFGeoTransMatrix = 34264; constexpr TagCodeType GeoTIFFGeoKeyDirectory = 34735; +constexpr TagCodeType GeoTIFFDoubleParams = 34736; constexpr TagCodeType GeoTIFFAsciiParams = 34737; // GDAL tags @@ -413,8 +424,20 @@ constexpr TagCodeType GDAL_NODATA = 42113; // GeoTIFF related constexpr TagCodeType RPCCoefficients = 50844; +// LERC compression related +constexpr TagCodeType LERCParameters = + 50674; /* Stores LERC version and additional compression method */ + } // namespace TagCode +/** Binary or'ed value of SubFileType flags */ +namespace SubFileTypeFlags +{ +constexpr uint32_t ReducedImage = 0x1; /* reduced resolution version */ +constexpr uint32_t Page = 0x2; /* one page of many */ +constexpr uint32_t Mask = 0x4; /* transparency mask */ +} // namespace SubFileTypeFlags + #define LIBERTIFF_CASE_TAGCODE_STR(x) \ case TagCode::x: \ return #x @@ -423,32 +446,42 @@ inline const char *tagCodeName(TagCodeType tagCode) { switch (tagCode) { - LIBERTIFF_CASE_TAGCODE_STR(NewSubfileType); - LIBERTIFF_CASE_TAGCODE_STR(SubfileType); + LIBERTIFF_CASE_TAGCODE_STR(SubFileType); + LIBERTIFF_CASE_TAGCODE_STR(OldSubFileType); LIBERTIFF_CASE_TAGCODE_STR(ImageWidth); LIBERTIFF_CASE_TAGCODE_STR(ImageLength); LIBERTIFF_CASE_TAGCODE_STR(BitsPerSample); LIBERTIFF_CASE_TAGCODE_STR(Compression); LIBERTIFF_CASE_TAGCODE_STR(PhotometricInterpretation); + LIBERTIFF_CASE_TAGCODE_STR(DocumentName); LIBERTIFF_CASE_TAGCODE_STR(ImageDescription); LIBERTIFF_CASE_TAGCODE_STR(StripOffsets); LIBERTIFF_CASE_TAGCODE_STR(SamplesPerPixel); LIBERTIFF_CASE_TAGCODE_STR(RowsPerStrip); LIBERTIFF_CASE_TAGCODE_STR(StripByteCounts); LIBERTIFF_CASE_TAGCODE_STR(PlanarConfiguration); + LIBERTIFF_CASE_TAGCODE_STR(Software); + LIBERTIFF_CASE_TAGCODE_STR(DateTime); LIBERTIFF_CASE_TAGCODE_STR(Predictor); + LIBERTIFF_CASE_TAGCODE_STR(ColorMap); LIBERTIFF_CASE_TAGCODE_STR(TileWidth); LIBERTIFF_CASE_TAGCODE_STR(TileLength); LIBERTIFF_CASE_TAGCODE_STR(TileOffsets); LIBERTIFF_CASE_TAGCODE_STR(TileByteCounts); + LIBERTIFF_CASE_TAGCODE_STR(ExtraSamples); LIBERTIFF_CASE_TAGCODE_STR(SampleFormat); + LIBERTIFF_CASE_TAGCODE_STR(Copyright); + LIBERTIFF_CASE_TAGCODE_STR(JPEGTables); LIBERTIFF_CASE_TAGCODE_STR(GeoTIFFPixelScale); LIBERTIFF_CASE_TAGCODE_STR(GeoTIFFTiePoints); + LIBERTIFF_CASE_TAGCODE_STR(GeoTIFFGeoTransMatrix); LIBERTIFF_CASE_TAGCODE_STR(GeoTIFFGeoKeyDirectory); + LIBERTIFF_CASE_TAGCODE_STR(GeoTIFFDoubleParams); LIBERTIFF_CASE_TAGCODE_STR(GeoTIFFAsciiParams); LIBERTIFF_CASE_TAGCODE_STR(GDAL_METADATA); LIBERTIFF_CASE_TAGCODE_STR(GDAL_NODATA); LIBERTIFF_CASE_TAGCODE_STR(RPCCoefficients); + LIBERTIFF_CASE_TAGCODE_STR(LERCParameters); default: break; } @@ -606,7 +639,23 @@ constexpr CompressionType CCITT_FAX4 = 4; constexpr CompressionType LZW = 5; constexpr CompressionType OldJPEG = 6; constexpr CompressionType JPEG = 7; -constexpr CompressionType Deflate = 8; +constexpr CompressionType Deflate = + 8; /* Deflate compression, as recognized by Adobe */ +constexpr CompressionType PackBits = 32773; +constexpr CompressionType LegacyDeflate = + 32946; /* Deflate compression, legacy tag */ +constexpr CompressionType JBIG = 34661; /* ISO JBIG */ +constexpr CompressionType LERC = + 34887; /* ESRI Lerc codec: https://github.com/Esri/lerc */ +constexpr CompressionType LZMA = 34925; /* LZMA2 */ +constexpr CompressionType ZSTD = + 50000; /* ZSTD: WARNING not registered in Adobe-maintained registry */ +constexpr CompressionType WEBP = + 50001; /* WEBP: WARNING not registered in Adobe-maintained registry */ +constexpr CompressionType JXL = + 50002; /* JPEGXL: WARNING not registered in Adobe-maintained registry */ +constexpr CompressionType JXL_DNG_1_7 = + 52546; /* JPEGXL from DNG 1.7 specification */ } // namespace Compression #define LIBERTIFF_CASE_COMPRESSION_STR(x) \ @@ -625,6 +674,15 @@ inline const char *compressionName(CompressionType compression) LIBERTIFF_CASE_COMPRESSION_STR(OldJPEG); LIBERTIFF_CASE_COMPRESSION_STR(JPEG); LIBERTIFF_CASE_COMPRESSION_STR(Deflate); + LIBERTIFF_CASE_COMPRESSION_STR(PackBits); + LIBERTIFF_CASE_COMPRESSION_STR(LegacyDeflate); + LIBERTIFF_CASE_COMPRESSION_STR(JBIG); + LIBERTIFF_CASE_COMPRESSION_STR(LERC); + LIBERTIFF_CASE_COMPRESSION_STR(LZMA); + LIBERTIFF_CASE_COMPRESSION_STR(ZSTD); + LIBERTIFF_CASE_COMPRESSION_STR(WEBP); + LIBERTIFF_CASE_COMPRESSION_STR(JXL); + LIBERTIFF_CASE_COMPRESSION_STR(JXL_DNG_1_7); default: break; } @@ -669,6 +727,17 @@ inline const char *sampleFormatName(SampleFormatType sampleFormat) #undef LIBERTIFF_CASE_SAMPLE_FORMAT_STR +/** Type of a ExtraSamples value */ +typedef uint32_t ExtraSamplesType; + +/** Values of the ExtraSamples tag */ +namespace ExtraSamples +{ +constexpr ExtraSamplesType Unspecified = 0; +constexpr ExtraSamplesType AssociatedAlpha = 1; /* premultiplied */ +constexpr ExtraSamplesType UnAssociatedAlpha = 2; /* unpremultiplied */ +} // namespace ExtraSamples + /** Content of a tag entry in a Image File Directory (IFD) */ struct TagEntry { @@ -893,6 +962,12 @@ class Image return m_nextImageOffset; } + /** Return value of SubFileType tag */ + inline uint32_t subFileType() const + { + return m_subFileType; + } + /** Return width of the image in pixels */ inline uint32_t width() const { @@ -953,6 +1028,12 @@ class Image return m_rowsPerStrip; } + /** Return the sanitized number of rows per strip */ + inline uint32_t rowsPerStripSanitized() const + { + return std::min(m_rowsPerStrip, m_height); + } + /** Return the number of strips/tiles. * Return 0 if inconsistent values between ByteCounts and Offsets arrays. */ inline uint64_t strileCount() const @@ -1234,6 +1315,7 @@ class Image std::set m_alreadyVisitedImageOffsets{}; uint64_t m_offset = 0; uint64_t m_nextImageOffset = 0; + uint32_t m_subFileType = 0; uint32_t m_width = 0; uint32_t m_height = 0; uint32_t m_bitsPerSample = 0; @@ -1268,6 +1350,10 @@ class Image { switch (entry.tag) { + case TagCode::SubFileType: + m_subFileType = singleValue; + break; + case TagCode::ImageWidth: m_width = singleValue; break;