From ed19141bca8f0c317e5ca71af1297cb4a17844d0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 24 Mar 2024 18:09:34 +0100 Subject: [PATCH 01/12] GTiff + internal libtiff: deal with issues with multi-band PlanarConfig=Contig, LERC and NaN values (fixes #9530) The LERC codec up-to-now didn't do any special processing when writing a strile in a multi-band PlanarConfig=Contig file, resulting in corruptions. Now we generate a mask buffer in that situation. Actually, if the NaN values are not at the same location accross bands, we need to write one mask-per-band, which requires liblerc >= 3.0, and in that situation change the LERC blob to multi-band instead of multi-depth/dim, with liblerc >= 3.0. When all bands have NaN values at the same location, we generate a single mask, which is compatible of liblerc < 3.0, but requires at last this version of libtiff to be read. This is the best solution we come with to fix the issue while having the maximum backward compatibility. This also fixes a wrong condition for creation of tiled JPEG where the condition to pad the right-most and bottom-most tiles was wrongly written and would have only worked for rasters with same number of tiles in horizontal and vertical dimension --- autotest/gcore/tiff_write.py | 151 +++++++-- frmts/gtiff/CMakeLists.txt | 11 + frmts/gtiff/geotiff.cpp | 13 + frmts/gtiff/gtiffdataset.h | 4 + frmts/gtiff/gtiffdataset_write.cpp | 160 +++++++-- frmts/gtiff/libtiff/tif_lerc.c | 507 ++++++++++++++++++++++++----- 6 files changed, 704 insertions(+), 142 deletions(-) diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index 5d1af8619af4..84ce3d3aa4cf 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -414,20 +414,8 @@ def test_tiff_write_13(): cs = ds.GetRasterBand(3).Checksum() ds = None - size = os.stat("tmp/sasha.tif").st_size - gdaltest.tiff_drv.Delete("tmp/sasha.tif") - assert cs in ( - 17347, - 14445, - 14135, # libjpeg 9e - ) - - md = gdaltest.tiff_drv.GetMetadata() - if md["LIBTIFF"] == "INTERNAL": - # 22816 with libjpeg-6b or libjpeg-turbo - # 22828 with libjpeg-9d - assert size <= 22828, "fail: bad size" + assert cs in (16612,) ############################################################################### @@ -9415,29 +9403,144 @@ def test_tiff_write_lerc_float(gdalDataType, structType): ############################################################################### -# Test LERC compression withFloat32/Float64 and nan + + +def lerc_version_at_least_3(): + + LERC_VERSION_MAJOR = gdal.GetDriverByName("GTiff").GetMetadataItem( + "LERC_VERSION_MAJOR", "LERC" + ) + return LERC_VERSION_MAJOR and int(LERC_VERSION_MAJOR) >= 3 + + +############################################################################### +# Test LERC compression with Float32/Float64 and nan @pytest.mark.parametrize( - "gdalDataType,structType", [[gdal.GDT_Float32, "f"], [gdal.GDT_Float64, "d"]] + "gdalDataType,structType", + [ + (gdal.GDT_Float32, "f"), + (gdal.GDT_Float64, "d"), + ], +) +@pytest.mark.parametrize("repeat", [1, 100]) +@pytest.mark.parametrize("interleave", ["PIXEL", "BAND"]) +@pytest.mark.parametrize( + "values", + [ + [(0.5,)], + [(0.5, 1.5)], + [(float("nan"),)], + [(0.5, float("nan"))], + [(0.5, 0.75, 1), (1.5, 1.75, 2)], + [(float("nan"),), (float("nan"),)], + [(0.5, float("nan"), 1), (1.5, float("nan"), 2)], + [ + (0.5, float("nan"), 1), + (1.5, 1.75, float("nan")), + ], # This one requires liblerc >= 3.0 since we need multiple masks + ], ) @pytest.mark.require_creation_option("GTiff", "LERC") -def test_tiff_write_lerc_float_with_nan(gdalDataType, structType): +def test_tiff_write_lerc_float_with_nan( + gdalDataType, structType, values, repeat, interleave +): - src_ds = gdal.GetDriverByName("MEM").Create("", 2, 1, 1, gdalDataType) - src_ds.GetRasterBand(1).WriteRaster( - 0, 0, 2, 1, struct.pack(structType * 2, 0.5, float("nan")) - ) + bandCount = len(values) + + if ( + bandCount == 2 + and True in [math.isnan(x) for x in values[0]] + and not ( + check_libtiff_internal_or_at_least(4, 6, 1) and lerc_version_at_least_3() + ) + ): + pytest.skip( + "multiple band with NaN in same strile only supported if libtiff >= 4.6.1 and liblerc >= 3.0" + ) + + width = len(values[0] * repeat) + src_ds = gdal.GetDriverByName("MEM").Create("", width, 1, bandCount, gdalDataType) + for i in range(bandCount): + src_ds.GetRasterBand(i + 1).WriteRaster( + 0, 0, width, 1, array.array(structType, values[i] * repeat).tobytes() + ) filename = "/vsimem/test.tif" - gdaltest.tiff_drv.CreateCopy(filename, src_ds, options=["COMPRESS=LERC"]) + gdaltest.tiff_drv.CreateCopy( + filename, src_ds, options=["COMPRESS=LERC", "INTERLEAVE=" + interleave] + ) ds = gdal.Open(filename) - got_data = struct.unpack(structType * 2, ds.ReadRaster()) - assert got_data[0] == 0.5 - assert math.isnan(got_data[1]) + for i in range(bandCount): + got_data = struct.unpack( + structType * width, ds.GetRasterBand(i + 1).ReadRaster() + ) + for j in range(width): + if math.isnan((values[i] * repeat)[j]): + assert math.isnan(got_data[j]) + else: + assert got_data[j] == (values[i] * repeat)[j] ds = None gdal.Unlink(filename) +############################################################################### + + +@pytest.mark.parametrize("tiled", [False, True]) +@pytest.mark.require_creation_option("GTiff", "LERC") +def test_tiff_write_lerc_float_with_nan_random(tmp_vsimem, tiled): + + """Stress test the floating-point LERC encoder, with several masks per strile""" + + width = 128 + height = 128 + bands = 100 + src_ds = gdal.GetDriverByName("MEM").Create( + "", width, height, bands, gdal.GDT_Float32 + ) + + import random + + band_values = [] + for i in range(bands): + # Generate random float values, but with at least 1/3 of nan in them in + # some bands and 2/3 in others + values = [int(random.random() * ((1 << 32) - 1)) for _ in range(width * height)] + values = array.array("I", values).tobytes() + values = [ + x if random.random() > (0.33 if (i % 2) == 0 else 0.67) else float("nan") + for x in struct.unpack("f" * (width * height), values) + ] + band_values.append(values) + values = array.array("f", values).tobytes() + src_ds.GetRasterBand(i + 1).WriteRaster(0, 0, width, height, values) + + filename = str(tmp_vsimem / "test_tiff_write_lerc_float_with_nan_random.tif") + if tiled: + options = ["COMPRESS=LERC", "TILED=YES", "BLOCKXSIZE=96", "BLOCKYSIZE=112"] + else: + options = ["COMPRESS=LERC", "BLOCKYSIZE=112"] + gdaltest.tiff_drv.CreateCopy(filename, src_ds, options=options) + + if not (check_libtiff_internal_or_at_least(4, 6, 1) and lerc_version_at_least_3()): + pytest.skip( + "multiple band with NaN in same strile only supported if libtiff >= 4.6.1 and liblerc >= 3.0" + ) + + ds = gdal.Open(filename) + for i in range(bands): + got_data = struct.unpack( + "f" * (width * height), ds.GetRasterBand(i + 1).ReadRaster() + ) + for j in range(width * height): + if math.isnan(band_values[i][j]): + assert math.isnan(got_data[j]) + else: + assert got_data[j] == band_values[i][j] + ds = None + + ############################################################################### # Test JXL compression diff --git a/frmts/gtiff/CMakeLists.txt b/frmts/gtiff/CMakeLists.txt index 8ea691ac0c83..575da527ea08 100644 --- a/frmts/gtiff/CMakeLists.txt +++ b/frmts/gtiff/CMakeLists.txt @@ -70,6 +70,12 @@ if (GDAL_USE_GEOTIFF_INTERNAL) target_compile_definitions(gdal_GTIFF PRIVATE -DINTERNAL_LIBGEOTIFF) endif () +if (GDAL_USE_LERC_INTERNAL) + # Just for the sake of reporting the liblerc version in internal metadata + target_compile_definitions(gdal_GTIFF PRIVATE -DLERC_SUPPORT) + gdal_add_vendored_lib(gdal_GTIFF lerc) +endif () + # Now deal with external dependencies # Include libjpeg first so that if we want to use jpeg-turbo on homebrew @@ -136,6 +142,11 @@ if (GDAL_USE_WEBP) target_include_directories(gdal_GTIFF PRIVATE $) endif () +if (NOT GDAL_USE_LERC_INTERNAL AND GDAL_USE_LERC) + target_compile_definitions(gdal_GTIFF PRIVATE -DLERC_SUPPORT) + target_include_directories(gdal_GTIFF PRIVATE $) +endif () + if(GDAL_USE_JPEG OR GDAL_USE_JPEG_INTERNAL) add_executable(generate_quant_table_md5sum EXCLUDE_FROM_ALL generate_quant_table_md5sum.cpp) gdal_standard_includes(generate_quant_table_md5sum) diff --git a/frmts/gtiff/geotiff.cpp b/frmts/gtiff/geotiff.cpp index c94ce9b18c59..7102d1adb2d5 100644 --- a/frmts/gtiff/geotiff.cpp +++ b/frmts/gtiff/geotiff.cpp @@ -46,6 +46,10 @@ #include "webp/encode.h" #endif +#ifdef LERC_SUPPORT +#include "Lerc_c_api.h" +#endif + #define STRINGIFY(x) #x #define XSTRINGIFY(x) STRINGIFY(x) @@ -1586,6 +1590,15 @@ void GDALRegister_GTiff() poDriver->SetMetadataItem("LIBGEOTIFF", XSTRINGIFY(LIBGEOTIFF_VERSION)); +#if defined(LERC_SUPPORT) && defined(LERC_VERSION_MAJOR) + poDriver->SetMetadataItem("LERC_VERSION_MAJOR", + XSTRINGIFY(LERC_VERSION_MAJOR), "LERC"); + poDriver->SetMetadataItem("LERC_VERSION_MINOR", + XSTRINGIFY(LERC_VERSION_MINOR), "LERC"); + poDriver->SetMetadataItem("LERC_VERSION_PATCH", + XSTRINGIFY(LERC_VERSION_PATCH), "LERC"); +#endif + poDriver->SetMetadataItem(GDAL_DCAP_COORDINATE_EPOCH, "YES"); poDriver->pfnOpen = GTiffDataset::Open; diff --git a/frmts/gtiff/gtiffdataset.h b/frmts/gtiff/gtiffdataset.h index 3198b5251902..7fb575ba444b 100644 --- a/frmts/gtiff/gtiffdataset.h +++ b/frmts/gtiff/gtiffdataset.h @@ -346,6 +346,10 @@ class GTiffDataset final : public GDALPamDataset bool WriteEncodedStrip(uint32_t strip, GByte *pabyData, int bPreserveDataBuffer); + template + void WriteDealWithLercAndNan(T *pBuffer, int nActualBlockWidth, + int nActualBlockHeight, int nStrileHeight); + bool HasOnlyNoData(const void *pBuffer, int nWidth, int nHeight, int nLineStride, int nComponents); inline bool IsFirstPixelEqualToNoData(const void *pBuffer); diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index c217ec096605..ed31eb82b28d 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -62,6 +62,11 @@ #include "tifvsi.h" #include "xtiffio.h" +#if LIFFLIB_VERSION > 20230908 || defined(INTERNAL_LIBTIFF) +/* libtiff < 4.6.1 doesn't generate a LERC mask for multi-band contig configuration */ +#define LIBTIFF_MULTIBAND_LERC_NAN_OK +#endif + static const int knGTIFFJpegTablesModeDefault = JPEGTABLESMODE_QUANT; static constexpr const char szPROFILE_BASELINE[] = "BASELINE"; @@ -532,6 +537,68 @@ inline bool GTiffDataset::IsFirstPixelEqualToNoData(const void *pBuffer) return false; } +/************************************************************************/ +/* WriteDealWithLercAndNan() */ +/************************************************************************/ + +template +void GTiffDataset::WriteDealWithLercAndNan(T *pBuffer, int nActualBlockWidth, + int nActualBlockHeight, + int nStrileHeight) +{ + // This method does 2 things: + // - warn the user if he tries to write NaN values with libtiff < 4.6.1 + // and multi-band PlanarConfig=Contig configuration + // - and in right-most and bottom-most tiles, replace non accessible + // pixel values by a safe one. + + const auto fPaddingValue = +#if !defined(LIBTIFF_MULTIBAND_LERC_NAN_OK) + m_nPlanarConfig == PLANARCONFIG_CONTIG && nBands > 1 + ? 0 + : +#endif + std::numeric_limits::quiet_NaN(); + + const int nBandsPerStrile = + m_nPlanarConfig == PLANARCONFIG_CONTIG ? nBands : 1; + for (int j = 0; j < nActualBlockHeight; ++j) + { +#if !defined(LIBTIFF_MULTIBAND_LERC_NAN_OK) + static bool bHasWarned = false; + if (m_nPlanarConfig == PLANARCONFIG_CONTIG && nBands > 1 && !bHasWarned) + { + for (int i = 0; i < nActualBlockWidth * nBandsPerStrile; ++i) + { + if (std::isnan( + pBuffer[j * m_nBlockXSize * nBandsPerStrile + i])) + { + bHasWarned = true; + CPLError(CE_Warning, CPLE_AppDefined, + "libtiff < 4.6.1 does not handle properly NaN " + "values for multi-band PlanarConfig=Contig " + "configuration. As a workaround, you can set the " + "INTERLEAVE=BAND creation option."); + break; + } + } + } +#endif + for (int i = nActualBlockWidth * nBandsPerStrile; + i < m_nBlockXSize * nBandsPerStrile; ++i) + { + pBuffer[j * m_nBlockXSize * nBandsPerStrile + i] = fPaddingValue; + } + } + for (int j = nActualBlockHeight; j < nStrileHeight; ++j) + { + for (int i = 0; i < m_nBlockXSize * nBandsPerStrile; ++i) + { + pBuffer[j * m_nBlockXSize * nBandsPerStrile + i] = fPaddingValue; + } + } +} + /************************************************************************/ /* WriteEncodedTile() */ /************************************************************************/ @@ -539,8 +606,16 @@ inline bool GTiffDataset::IsFirstPixelEqualToNoData(const void *pBuffer) bool GTiffDataset::WriteEncodedTile(uint32_t tile, GByte *pabyData, int bPreserveDataBuffer) { - int iRow = 0; - int iColumn = 0; + + const int iColumn = (tile % m_nBlocksPerBand) % m_nBlocksPerRow; + const int iRow = (tile % m_nBlocksPerBand) / m_nBlocksPerRow; + + const int nActualBlockWidth = (iColumn == m_nBlocksPerRow - 1) + ? nRasterXSize - iColumn * m_nBlockXSize + : m_nBlockXSize; + const int nActualBlockHeight = (iRow == m_nBlocksPerColumn - 1) + ? nRasterYSize - iRow * m_nBlockYSize + : m_nBlockYSize; /* -------------------------------------------------------------------- */ /* Don't write empty blocks in some cases. */ @@ -552,18 +627,6 @@ bool GTiffDataset::WriteEncodedTile(uint32_t tile, GByte *pabyData, const int nComponents = m_nPlanarConfig == PLANARCONFIG_CONTIG ? nBands : 1; - iColumn = (tile % m_nBlocksPerBand) % m_nBlocksPerRow; - iRow = (tile % m_nBlocksPerBand) / m_nBlocksPerRow; - - const int nActualBlockWidth = - (iColumn == m_nBlocksPerRow - 1) - ? nRasterXSize - iColumn * m_nBlockXSize - : m_nBlockXSize; - const int nActualBlockHeight = - (iRow == m_nBlocksPerColumn - 1) - ? nRasterYSize - iRow * m_nBlockYSize - : m_nBlockYSize; - if (HasOnlyNoData(pabyData, nActualBlockWidth, nActualBlockHeight, m_nBlockXSize, nComponents)) { @@ -572,23 +635,21 @@ bool GTiffDataset::WriteEncodedTile(uint32_t tile, GByte *pabyData, } } - // Do we need to spread edge values right or down for a partial - // JPEG encoded tile? We do this to avoid edge artifacts. - bool bNeedTileFill = false; - if (m_nCompression == COMPRESSION_JPEG) - { - iColumn = (tile % m_nBlocksPerBand) % m_nBlocksPerRow; - iRow = (tile % m_nBlocksPerBand) / m_nBlocksPerRow; + // Is this a partial right edge or bottom edge tile? + const bool bPartialTile = (nActualBlockWidth < m_nBlockXSize) || + (nActualBlockHeight < m_nBlockYSize); - // Is this a partial right edge tile? - if (iRow == m_nBlocksPerRow - 1 && nRasterXSize % m_nBlockXSize != 0) - bNeedTileFill = true; + const bool bIsLercFloatingPoint = + m_nCompression == COMPRESSION_LERC && + (GetRasterBand(1)->GetRasterDataType() == GDT_Float32 || + GetRasterBand(1)->GetRasterDataType() == GDT_Float64); - // Is this a partial bottom edge tile? - if (iColumn == m_nBlocksPerColumn - 1 && - nRasterYSize % m_nBlockYSize != 0) - bNeedTileFill = true; - } + // Do we need to spread edge values right or down for a partial + // JPEG encoded tile? We do this to avoid edge artifacts. + // We also need to be careful with LERC and NaN values + const bool bNeedTempBuffer = + bPartialTile && + (m_nCompression == COMPRESSION_JPEG || bIsLercFloatingPoint); // If we need to fill out the tile, or if we want to prevent // TIFFWriteEncodedTile from altering the buffer as part of @@ -597,7 +658,7 @@ bool GTiffDataset::WriteEncodedTile(uint32_t tile, GByte *pabyData, const GPtrDiff_t cc = static_cast(TIFFTileSize(m_hTIFF)); if (bPreserveDataBuffer && - (TIFFIsByteSwapped(m_hTIFF) || bNeedTileFill || m_panMaskOffsetLsb)) + (TIFFIsByteSwapped(m_hTIFF) || bNeedTempBuffer || m_panMaskOffsetLsb)) { if (m_pabyTempWriteBuffer == nullptr) { @@ -611,7 +672,8 @@ bool GTiffDataset::WriteEncodedTile(uint32_t tile, GByte *pabyData, // Perform tile fill if needed. // TODO: we should also handle the case of nBitsPerSample == 12 // but this is more involved. - if (bNeedTileFill && m_nBitsPerSample == 8) + if (bPartialTile && m_nCompression == COMPRESSION_JPEG && + m_nBitsPerSample == 8) { const int nComponents = m_nPlanarConfig == PLANARCONFIG_CONTIG ? nBands : 1; @@ -656,6 +718,24 @@ bool GTiffDataset::WriteEncodedTile(uint32_t tile, GByte *pabyData, } } + if (bIsLercFloatingPoint && + (bPartialTile +#if !defined(LIBTIFF_MULTIBAND_LERC_NAN_OK) + /* libtiff < 4.6.1 doesn't generate a LERC mask for multi-band contig configuration */ + || (m_nPlanarConfig == PLANARCONFIG_CONTIG && nBands > 1) +#endif + )) + { + if (GetRasterBand(1)->GetRasterDataType() == GDT_Float32) + WriteDealWithLercAndNan(reinterpret_cast(pabyData), + nActualBlockWidth, nActualBlockHeight, + m_nBlockYSize); + else + WriteDealWithLercAndNan(reinterpret_cast(pabyData), + nActualBlockWidth, nActualBlockHeight, + m_nBlockYSize); + } + if (m_panMaskOffsetLsb) { const int iBand = m_nPlanarConfig == PLANARCONFIG_SEPARATE @@ -757,6 +837,24 @@ bool GTiffDataset::WriteEncodedStrip(uint32_t strip, GByte *pabyData, pabyData = static_cast(m_pabyTempWriteBuffer); } +#if !defined(LIBTIFF_MULTIBAND_LERC_NAN_OK) + const bool bIsLercFloatingPoint = + m_nCompression == COMPRESSION_LERC && + (GetRasterBand(1)->GetRasterDataType() == GDT_Float32 || + GetRasterBand(1)->GetRasterDataType() == GDT_Float64); + if (bIsLercFloatingPoint && + /* libtiff < 4.6.1 doesn't generate a LERC mask for multi-band contig configuration */ + m_nPlanarConfig == PLANARCONFIG_CONTIG && nBands > 1) + { + if (GetRasterBand(1)->GetRasterDataType() == GDT_Float32) + WriteDealWithLercAndNan(reinterpret_cast(pabyData), + m_nBlockXSize, nStripHeight, nStripHeight); + else + WriteDealWithLercAndNan(reinterpret_cast(pabyData), + m_nBlockXSize, nStripHeight, nStripHeight); + } +#endif + if (m_panMaskOffsetLsb) { int iBand = m_nPlanarConfig == PLANARCONFIG_SEPARATE diff --git a/frmts/gtiff/libtiff/tif_lerc.c b/frmts/gtiff/libtiff/tif_lerc.c index b2a3eb8a4bcf..d9d8d3342eb3 100644 --- a/frmts/gtiff/libtiff/tif_lerc.c +++ b/frmts/gtiff/libtiff/tif_lerc.c @@ -71,6 +71,9 @@ typedef struct uint8_t *uncompressed_buffer; unsigned int uncompressed_offset; + uint8_t *uncompressed_buffer_multiband; + unsigned int uncompressed_buffer_multiband_alloc; + unsigned int mask_size; uint8_t *mask_buffer; @@ -168,7 +171,7 @@ static int GetLercDataType(TIFF *tif) return -1; } -static int SetupUncompressedBuffer(TIFF *tif, LERCState *sp, const char *module) +static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) { TIFFDirectory *td = &tif->tif_dir; uint64_t new_size_64; @@ -202,8 +205,9 @@ static int SetupUncompressedBuffer(TIFF *tif, LERCState *sp, const char *module) sp->uncompressed_size = new_size; /* add some margin as we are going to use it also to store deflate/zstd - * compressed data */ - new_alloc_64 = 100 + new_size_64 + new_size_64 / 3; + * compressed data. We also need extra margin when writing very small + * rasters with one mask per band. */ + new_alloc_64 = 256 + new_size_64 + new_size_64 / 3; #ifdef ZSTD_SUPPORT { size_t zstd_max = ZSTD_compressBound((size_t)new_size_64); @@ -243,11 +247,17 @@ static int SetupUncompressedBuffer(TIFF *tif, LERCState *sp, const char *module) td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1) || (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && - (td->td_planarconfig == PLANARCONFIG_SEPARATE || - td->td_samplesperpixel == 1) && (td->td_bitspersample == 32 || td->td_bitspersample == 64))) { unsigned int mask_size = sp->segment_width * sp->segment_height; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG) + { + /* We may need one mask per band */ + mask_size *= td->td_samplesperpixel; + } +#endif if (sp->mask_size < mask_size) { void *mask_buffer = @@ -279,7 +289,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) TIFFDirectory *td = &tif->tif_dir; LERCState *sp = LERCDecoderState(tif); int lerc_data_type; - unsigned int infoArray[8]; + unsigned int infoArray[9]; unsigned nomask_bands = td->td_samplesperpixel; int ndims; int use_mask = 0; @@ -295,7 +305,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) if (lerc_data_type < 0) return 0; - if (!SetupUncompressedBuffer(tif, sp, module)) + if (!SetupBuffers(tif, sp, module)) return 0; if (sp->additional_compression != LERC_ADD_COMPRESSION_NONE) @@ -400,7 +410,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } lerc_ret = - lerc_getBlobInfo(lerc_data, lerc_data_size, infoArray, NULL, 8, 0); + lerc_getBlobInfo(lerc_data, lerc_data_size, infoArray, NULL, 9, 0); if (lerc_ret != 0) { TIFFErrorExtR(tif, module, "lerc_getBlobInfo() failed"); @@ -418,18 +428,16 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) use_mask = 1; nomask_bands--; } - else if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && - (td->td_planarconfig == PLANARCONFIG_SEPARATE || - td->td_samplesperpixel == 1) && - (td->td_bitspersample == 32 || td->td_bitspersample == 64)) + else if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP) { use_mask = 1; } ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ? nomask_bands : 1; - /* Info returned in infoArray is { version, dataType, nDim, nCols, - nRows, nBands, nValidPixels, blobSize } */ + /* Info returned in infoArray is { version, dataType, nDim/nDepth, nCols, + nRows, nBands, nValidPixels, blobSize, + and starting with liblerc 3.0 nRequestedMasks } */ if (infoArray[0] != (unsigned)sp->lerc_version) { TIFFWarningExtR(tif, module, @@ -442,12 +450,29 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) infoArray[1], lerc_data_type); return 0; } - if (infoArray[2] != (unsigned)ndims) + + const unsigned nFoundDims = infoArray[2]; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG && + td->td_samplesperpixel > 1) + { + if (nFoundDims != 1 && nFoundDims != (unsigned)ndims) + { + TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: 1 or %d", + nFoundDims, ndims); + return 0; + } + } + else +#endif + if (nFoundDims != (unsigned)ndims) { TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: %d", - infoArray[2], ndims); + nFoundDims, ndims); return 0; } + if (infoArray[3] != sp->segment_width) { TIFFErrorExtR(tif, module, "Unexpected nCols: %d. Expected: %du", @@ -460,12 +485,38 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) infoArray[4], sp->segment_height); return 0; } - if (infoArray[5] != 1) + + const unsigned nFoundBands = infoArray[5]; + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG && + td->td_samplesperpixel > 1 && nFoundDims == 1) + { +#if !LERC_AT_LEAST_VERSION(3, 0, 0) + if (nFoundBands == td->td_samplesperpixel) + { + TIFFErrorExtR( + tif, module, + "Unexpected nBands: %d. This file may have been generated with " + "a liblerc version >= 3.0, with one mask per band, and is not " + "supported by this older version of liblerc", + nFoundBands); + return 0; + } +#endif + if (nFoundBands != td->td_samplesperpixel) + { + TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", + nFoundBands, td->td_samplesperpixel); + return 0; + } + } + else if (nFoundBands != 1) { TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", - infoArray[5], 1); + nFoundBands, 1); return 0; } + if (infoArray[7] != lerc_data_size) { TIFFErrorExtR(tif, module, "Unexpected blobSize: %d. Expected: %u", @@ -473,13 +524,75 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) return 0; } - lerc_ret = lerc_decode(lerc_data, lerc_data_size, + int nRequestedMasks = use_mask ? 1 : 0; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + const int nFoundMasks = infoArray[8]; + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && + td->td_planarconfig == PLANARCONFIG_CONTIG && + td->td_samplesperpixel > 1 && nFoundDims == 1) + { + if (nFoundMasks != 0 && nFoundMasks != td->td_samplesperpixel) + { + TIFFErrorExtR(tif, module, + "Unexpected nFoundMasks: %d. Expected: 0 or %d", + nFoundMasks, td->td_samplesperpixel); + return 0; + } + nRequestedMasks = nFoundMasks; + } + else + { + if (nFoundMasks != 0 && nFoundMasks != 1) + { + TIFFErrorExtR(tif, module, + "Unexpected nFoundMasks: %d. Expected: 0 or 1", + nFoundMasks); + return 0; + } + } + if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && nFoundMasks == 0) + { + nRequestedMasks = 0; + use_mask = 0; + } +#endif + + const unsigned nb_pixels = sp->segment_width * sp->segment_height; + #if LERC_AT_LEAST_VERSION(3, 0, 0) - use_mask ? 1 : 0, + if (nRequestedMasks > 1) + { + unsigned int num_bytes_needed = + nb_pixels * td->td_samplesperpixel * (td->td_bitspersample / 8); + if (sp->uncompressed_buffer_multiband_alloc < num_bytes_needed) + { + _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); + sp->uncompressed_buffer_multiband = + _TIFFmallocExt(tif, num_bytes_needed); + if (!sp->uncompressed_buffer_multiband) + { + sp->uncompressed_buffer_multiband_alloc = 0; + return 0; + } + sp->uncompressed_buffer_multiband_alloc = num_bytes_needed; + } + lerc_ret = lerc_decode(lerc_data, lerc_data_size, nRequestedMasks, + sp->mask_buffer, nFoundDims, sp->segment_width, + sp->segment_height, nFoundBands, lerc_data_type, + sp->uncompressed_buffer_multiband); + } + else #endif - use_mask ? sp->mask_buffer : NULL, ndims, - sp->segment_width, sp->segment_height, 1, - lerc_data_type, sp->uncompressed_buffer); + { + lerc_ret = + lerc_decode(lerc_data, lerc_data_size, +#if LERC_AT_LEAST_VERSION(3, 0, 0) + nRequestedMasks, +#endif + use_mask ? sp->mask_buffer : NULL, nFoundDims, + sp->segment_width, sp->segment_height, nFoundBands, + lerc_data_type, sp->uncompressed_buffer); + } if (lerc_ret != 0) { TIFFErrorExtR(tif, module, "lerc_decode() failed"); @@ -515,7 +628,6 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } else if (use_mask && td->td_sampleformat == SAMPLEFORMAT_IEEEFP) { - const unsigned nb_pixels = sp->segment_width * sp->segment_height; unsigned i; #if WORDS_BIGENDIAN const unsigned char nan_bytes[] = {0x7f, 0xc0, 0, 0}; @@ -525,23 +637,104 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) float nan_float32; memcpy(&nan_float32, nan_bytes, 4); - if (td->td_bitspersample == 32) + if (td->td_planarconfig == PLANARCONFIG_SEPARATE || + td->td_samplesperpixel == 1) { - for (i = 0; i < nb_pixels; i++) + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + if (sp->mask_buffer[i] == 0) + ((float *)sp->uncompressed_buffer)[i] = nan_float32; + } + } + else + { + const double nan_float64 = nan_float32; + for (i = 0; i < nb_pixels; i++) + { + if (sp->mask_buffer[i] == 0) + ((double *)sp->uncompressed_buffer)[i] = nan_float64; + } + } + } + else if (nRequestedMasks == 1) + { + assert(nFoundDims == td->td_samplesperpixel); + assert(nFoundBands == 1); + + unsigned k = 0; + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i] == 0) + ((float *)sp->uncompressed_buffer)[k] = nan_float32; + ++k; + } + } + } + else { - if (sp->mask_buffer[i] == 0) - ((float *)sp->uncompressed_buffer)[i] = nan_float32; + const double nan_float64 = nan_float32; + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i] == 0) + ((double *)sp->uncompressed_buffer)[k] = + nan_float64; + ++k; + } + } } } +#if LERC_AT_LEAST_VERSION(3, 0, 0) else { - const double nan_float64 = nan_float32; - for (i = 0; i < nb_pixels; i++) + assert(nRequestedMasks == td->td_samplesperpixel); + assert(nFoundDims == 1); + assert(nFoundBands == td->td_samplesperpixel); + + unsigned k = 0; + if (td->td_bitspersample == 32) { - if (sp->mask_buffer[i] == 0) - ((double *)sp->uncompressed_buffer)[i] = nan_float64; + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i + j * nb_pixels] == 0) + ((float *)sp->uncompressed_buffer)[k] = nan_float32; + else + ((float *)sp->uncompressed_buffer)[k] = + ((float *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels]; + ++k; + } + } + } + else + { + const double nan_float64 = nan_float32; + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; j++) + { + if (sp->mask_buffer[i + j * nb_pixels] == 0) + ((double *)sp->uncompressed_buffer)[k] = + nan_float64; + else + ((double *)sp->uncompressed_buffer)[k] = + ((double *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels]; + ++k; + } + } } } +#endif } return 1; @@ -611,7 +804,7 @@ static int LERCPreEncode(TIFF *tif, uint16_t s) if (lerc_data_type < 0) return 0; - if (!SetupUncompressedBuffer(tif, sp, module)) + if (!SetupBuffers(tif, sp, module)) return 0; return 1; @@ -650,7 +843,6 @@ static int LERCPostEncode(TIFF *tif) lerc_status lerc_ret; static const char module[] = "LERCPostEncode"; LERCState *sp = LERCEncoderState(tif); - unsigned int numBytes = 0; unsigned int numBytesWritten = 0; TIFFDirectory *td = &tif->tif_dir; int use_mask = 0; @@ -662,6 +854,9 @@ static int LERCPostEncode(TIFF *tif) return 0; } + int mask_count = 1; + const unsigned nb_pixels = sp->segment_width * sp->segment_height; + /* Extract alpha mask (if containing only 0 and 255 values, */ /* and compact array of regular bands */ if (td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_extrasamples > 0 && @@ -673,7 +868,6 @@ static int LERCPostEncode(TIFF *tif) const unsigned src_stride = td->td_samplesperpixel * (td->td_bitspersample / 8); unsigned i = 0; - const unsigned nb_pixels = sp->segment_width * sp->segment_height; use_mask = 1; for (i = 0; i < nb_pixels; i++) @@ -710,109 +904,247 @@ static int LERCPostEncode(TIFF *tif) } } else if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && - (td->td_planarconfig == PLANARCONFIG_SEPARATE || - dst_nbands == 1) && (td->td_bitspersample == 32 || td->td_bitspersample == 64)) { /* Check for NaN values */ unsigned i; - const unsigned nb_pixels = sp->segment_width * sp->segment_height; if (td->td_bitspersample == 32) { - for (i = 0; i < nb_pixels; i++) + if (td->td_planarconfig == PLANARCONFIG_CONTIG && dst_nbands > 1) { - const float val = ((float *)sp->uncompressed_buffer)[i]; - if (val != val) + unsigned k = 0; + for (i = 0; i < nb_pixels; i++) { - use_mask = 1; - break; + int count_nan = 0; + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const float val = ((float *)sp->uncompressed_buffer)[k]; + ++k; + if (val != val) + { + ++count_nan; + } + } + if (count_nan > 0) + { + use_mask = 1; + if (count_nan < td->td_samplesperpixel) + { + mask_count = td->td_samplesperpixel; + break; + } + } + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + const float val = ((float *)sp->uncompressed_buffer)[i]; + if (val != val) + { + use_mask = 1; + break; + } } } } else { - for (i = 0; i < nb_pixels; i++) + if (td->td_planarconfig == PLANARCONFIG_CONTIG && dst_nbands > 1) { - const double val = ((double *)sp->uncompressed_buffer)[i]; - if (val != val) + unsigned k = 0; + for (i = 0; i < nb_pixels; i++) + { + int count_nan = 0; + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const double val = + ((double *)sp->uncompressed_buffer)[k]; + ++k; + if (val != val) + { + ++count_nan; + } + } + if (count_nan > 0) + { + use_mask = 1; + if (count_nan < td->td_samplesperpixel) + { + mask_count = td->td_samplesperpixel; + break; + } + } + } + } + else + { + for (i = 0; i < nb_pixels; i++) { - use_mask = 1; - break; + const double val = ((double *)sp->uncompressed_buffer)[i]; + if (val != val) + { + use_mask = 1; + break; + } } } } if (use_mask) { - if (td->td_bitspersample == 32) + if (mask_count > 1) { - for (i = 0; i < nb_pixels; i++) +#if LERC_AT_LEAST_VERSION(3, 0, 0) + unsigned int num_bytes_needed = + nb_pixels * dst_nbands * (td->td_bitspersample / 8); + if (sp->uncompressed_buffer_multiband_alloc < num_bytes_needed) { - const float val = ((float *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); + sp->uncompressed_buffer_multiband = + _TIFFmallocExt(tif, num_bytes_needed); + if (!sp->uncompressed_buffer_multiband) + { + sp->uncompressed_buffer_multiband_alloc = 0; + return 0; + } + sp->uncompressed_buffer_multiband_alloc = num_bytes_needed; + } + + unsigned k = 0; + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const float val = + ((float *)sp->uncompressed_buffer)[k]; + ((float *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels] = val; + ++k; + sp->mask_buffer[i + j * nb_pixels] = + (val == val) ? 255 : 0; + } + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + for (int j = 0; j < td->td_samplesperpixel; ++j) + { + const double val = + ((double *)sp->uncompressed_buffer)[k]; + ((double *)sp->uncompressed_buffer_multiband) + [i + j * nb_pixels] = val; + ++k; + sp->mask_buffer[i + j * nb_pixels] = + (val == val) ? 255 : 0; + } + } + } +#else + TIFFErrorExtR(tif, module, + "lerc_encode() would need to create one mask per " + "sample, but this requires liblerc >= 3.0"); + return 0; +#endif + } + else if (td->td_planarconfig == PLANARCONFIG_CONTIG && + dst_nbands > 1) + { + if (td->td_bitspersample == 32) + { + for (i = 0; i < nb_pixels; i++) + { + const float val = + ((float *)sp->uncompressed_buffer)[i * dst_nbands]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + const double val = + ((double *)sp->uncompressed_buffer)[i * dst_nbands]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } } } else { - for (i = 0; i < nb_pixels; i++) + if (td->td_bitspersample == 32) { - const double val = ((double *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + for (i = 0; i < nb_pixels; i++) + { + const float val = ((float *)sp->uncompressed_buffer)[i]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } + } + else + { + for (i = 0; i < nb_pixels; i++) + { + const double val = + ((double *)sp->uncompressed_buffer)[i]; + sp->mask_buffer[i] = (val == val) ? 255 : 0; + } } } } } -#if 0 - lerc_ret = lerc_computeCompressedSize( - sp->uncompressed_buffer, - sp->lerc_version, - GetLercDataType(tif), - td->td_planarconfig == PLANARCONFIG_CONTIG ? - dst_nbands : 1, - sp->segment_width, - sp->segment_height, - 1, - use_mask ? sp->mask_buffer : NULL, - sp->maxzerror, - &numBytes); - if( lerc_ret != 0 ) - { - TIFFErrorExtR(tif, module, - "lerc_computeCompressedSize() failed"); - return 0; - } -#else - numBytes = sp->uncompressed_alloc; + unsigned int estimated_compressed_size = sp->uncompressed_alloc; +#if LERC_AT_LEAST_VERSION(3, 0, 0) + if (mask_count > 1) + { + estimated_compressed_size += nb_pixels * mask_count / 8; + } #endif - if (sp->compressed_size < numBytes) + if (sp->compressed_size < estimated_compressed_size) { _TIFFfreeExt(tif, sp->compressed_buffer); - sp->compressed_buffer = _TIFFmallocExt(tif, numBytes); + sp->compressed_buffer = _TIFFmallocExt(tif, estimated_compressed_size); if (!sp->compressed_buffer) { sp->compressed_size = 0; return 0; } - sp->compressed_size = numBytes; + sp->compressed_size = estimated_compressed_size; } - lerc_ret = lerc_encodeForVersion( - sp->uncompressed_buffer, sp->lerc_version, GetLercDataType(tif), - td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1, - sp->segment_width, sp->segment_height, 1, #if LERC_AT_LEAST_VERSION(3, 0, 0) - use_mask ? 1 : 0, + if (mask_count > 1) + { + lerc_ret = lerc_encodeForVersion( + sp->uncompressed_buffer_multiband, sp->lerc_version, + GetLercDataType(tif), 1, sp->segment_width, sp->segment_height, + dst_nbands, dst_nbands, sp->mask_buffer, sp->maxzerror, + sp->compressed_buffer, sp->compressed_size, &numBytesWritten); + } + else #endif - use_mask ? sp->mask_buffer : NULL, sp->maxzerror, sp->compressed_buffer, - sp->compressed_size, &numBytesWritten); + { + lerc_ret = lerc_encodeForVersion( + sp->uncompressed_buffer, sp->lerc_version, GetLercDataType(tif), + td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1, + sp->segment_width, sp->segment_height, 1, +#if LERC_AT_LEAST_VERSION(3, 0, 0) + use_mask ? 1 : 0, +#endif + use_mask ? sp->mask_buffer : NULL, sp->maxzerror, + sp->compressed_buffer, sp->compressed_size, &numBytesWritten); + } if (lerc_ret != 0) { TIFFErrorExtR(tif, module, "lerc_encode() failed"); return 0; } - assert(numBytesWritten < numBytes); + assert(numBytesWritten < estimated_compressed_size); if (sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE) { @@ -958,6 +1290,7 @@ static void LERCCleanup(TIFF *tif) tif->tif_tagmethods.vsetfield = sp->vsetparent; _TIFFfreeExt(tif, sp->uncompressed_buffer); + _TIFFfreeExt(tif, sp->uncompressed_buffer_multiband); _TIFFfreeExt(tif, sp->compressed_buffer); _TIFFfreeExt(tif, sp->mask_buffer); From 1461a1ab7f2ce9c6765f49f2da4be58c0a965108 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 28 Mar 2024 15:36:40 +0100 Subject: [PATCH 02/12] CI: codeql.yml: initialize CodeQL after running cmake to avoid #9549 --- .github/workflows/codeql.yml | 79 ++++++++++-------------------------- 1 file changed, 21 insertions(+), 58 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 82f14ad60f9b..11d48db46979 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -43,22 +43,6 @@ jobs: - name: Checkout repository uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - config: | - query-filters: - - exclude: - id: cpp/non-https-url - - name: Install dependencies run: | sudo apt-get update @@ -104,55 +88,34 @@ jobs: libzstd-dev \ unixodbc-dev - # cache the .ccache directory - # key it on the runner os, build type, deps, and arch - # It's especially important to include arch in the key because we - # may get runtime errors with -mavx2 from objects built on a - # different architecture. - - name: Restore build cache + - name: Configure if: matrix.language == 'c-cpp' - id: restore-cache - uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + run: | + mkdir build + (cd build && cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DGDAL_USE_LERC_INTERNAL=OFF) + + # Initializes the CodeQL tools for scanning. + # We do that after running CMake to avoid CodeQL to trigger during CMake time, + # in particular during HDF5 detection which is terribly slow (https://github.com/OSGeo/gdal/issues/9549) + - name: Initialize CodeQL + uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 with: - path: ${{ github.workspace }}/.ccache - key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }} - restore-keys: | - ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }} - ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }} + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - - name: Configure ccache - if: matrix.language == 'c-cpp' - run: | - echo CCACHE_BASEDIR=${{ github.workspace }} >> ${GITHUB_ENV} - echo CCACHE_DIR=${{ github.workspace }}/.ccache >> ${GITHUB_ENV} - echo CCACHE_MAXSIZE=250M >> ${GITHUB_ENV} - ccache -z + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + config: | + query-filters: + - exclude: + id: cpp/non-https-url - name: Build if: matrix.language == 'c-cpp' run: | - mkdir build - cd build - # LD_PRELOAD is initially set to something like /opt/hostedtoolcache/CodeQL/2.16.5/x64/codeql/tools/linux64/lib64trace.so - # To avoid CodeQL to trigger during CMake time, in particular during HDF5 detection which is terribly slow (https://github.com/OSGeo/gdal/issues/9549), we temporarily rename that file - export LD_PRELOAD_RESOLVED=$(env | grep LD_PRELOAD | sed "s/LD_PRELOAD=//") - echo "$LD_PRELOAD_RESOLVED" - sudo mv "$LD_PRELOAD_RESOLVED" "$LD_PRELOAD_RESOLVED.disabled" - cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSE_CCACHE=YES -DGDAL_USE_LERC_INTERNAL=OFF - sudo mv "$LD_PRELOAD_RESOLVED.disabled" "$LD_PRELOAD_RESOLVED" - make -j$(nproc) - - - name: Summarize ccache - if: matrix.language == 'c-cpp' - run: | - ccache -s - - - name: Save build cache - if: matrix.language == 'c-cpp' - uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 - with: - path: ${{ github.workspace }}/.ccache - key: ${{ steps.restore-cache.outputs.cache-primary-key }} + (cd build && make -j$(nproc)) - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 From 04c2c1b290a7a2abb80fa66195eec08350a249cb Mon Sep 17 00:00:00 2001 From: Daniel Baston Date: Thu, 28 Mar 2024 20:37:21 -0400 Subject: [PATCH 03/12] Doc: Update vector data model description Add discussion of newer geometry types, feature styling, field types and subtypes, field domains, and relationships. --- doc/source/substitutions.rst | 3 +- doc/source/user/configoptions.rst | 2 + doc/source/user/vector_data_model.rst | 121 +++++++++++++++++--------- 3 files changed, 86 insertions(+), 40 deletions(-) diff --git a/doc/source/substitutions.rst b/doc/source/substitutions.rst index 6d8582f6ffff..ccc9f9c6fa55 100644 --- a/doc/source/substitutions.rst +++ b/doc/source/substitutions.rst @@ -1,5 +1,6 @@ .. |PDAL| replace:: `PDAL `__ -.. |Proj.4| replace:: `Proj.4 `__ +.. |Proj.4| replace:: `PROJ `__ +.. |PROJ| replace:: `PROJ `__ .. |GeoTIFF| replace:: `GeoTIFF `__ .. |QGIS| replace:: `QGIS `__ .. |GDAL| replace:: `GDAL `__ diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst index 957ab52059e1..e6088332853d 100644 --- a/doc/source/user/configoptions.rst +++ b/doc/source/user/configoptions.rst @@ -493,6 +493,8 @@ General options Location of Python shared library file, e.g. ``pythonX.Y[...].so/.dll``. +.. _configoptions_vector: + Vector related options ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/source/user/vector_data_model.rst b/doc/source/user/vector_data_model.rst index cf8b3b8020fa..e3f94cf82f0c 100644 --- a/doc/source/user/vector_data_model.rst +++ b/doc/source/user/vector_data_model.rst @@ -1,45 +1,51 @@ +.. include:: ../substitutions.rst + .. _vector_data_model: ================================================================================ Vector Data Model ================================================================================ -This document is intended to document the OGR classes. The OGR classes are intended to be generic (not specific to OLE DB or COM or Windows) but are used as a foundation for implementing OLE DB Provider support, as well as client side support for SFCOM. It is intended that these same OGR classes could be used by an implementation of SFCORBA for instance or used directly by C++ programs wanting to use an OpenGIS simple features inspired API. - -Because OGR is modeled on the OpenGIS simple features data model, it is very helpful to review the SFCOM, or other simple features interface specifications which can be retrieved from the Open Geospatial Consortium web site. Data types, and method names are modeled on those from the interface specifications. +This page documents the classes used to handle vector data. Many data types and method names are based on the OGC Simple Features data model, so it may be helpful to review the `specifications published by OGC `__. +For historical reasons, GDAL uses the "OGR" prefix to denote types and functions that apply only to vector data. Class Overview -------------- -- Geometry (:ref:`ogr_geometry.h `): The geometry classes (:cpp:class:`OGRGeometry`, etc) encapsulate the OpenGIS model vector data as well as providing some geometry operations, and translation to/from well known binary and text format. A geometry includes a spatial reference system (projection). +The following classes form the core of the vector data model: + +- Geometry (:ref:`ogr_geometry.h `): The geometry classes (:cpp:class:`OGRGeometry`, etc) encapsulate the OGC vector data types. They provide some geometry operations and translation to/from well known binary and text format. A geometry includes a spatial reference system (projection). - Spatial Reference (:ref:`ogr_spatialref.h `): An :cpp:class:`OGRSpatialReference` encapsulates the definition of a projection and datum. -- Feature (:ref:`ogr_feature.h `): The :cpp:class:`OGRFeature` encapsulates the definition of a whole feature, that is a geometry and a set of attributes. +- Feature (:ref:`ogr_feature.h `): The :cpp:class:`OGRFeature` encapsulates the definition of a whole feature, that is a set of geometries and attributes relating to a single entity. - Feature Class Definition (:ref:`ogr_feature.h `): The :cpp:class:`OGRFeatureDefn` class captures the schema (set of field definitions) for a group of related features (normally a whole layer). -- Layer (:ref:`ogrsf_frmts.h `): :cpp:class:`OGRLayer` is an abstract base class represent a layer of features in an GDALDataset. -- Dataset (:ref:`gdal_priv.h `): A :cpp:class:`GDALDataset` is an abstract base class representing a file or database containing one or more OGRLayer objects. -- Drivers (gdal_priv.h): A :cpp:class:`GDALDriver` represents a translator for a specific format, opening GDALDataset objects. All available drivers are managed by the GDALDriverManager. +- Layer (:ref:`ogrsf_frmts.h `): :cpp:class:`OGRLayer` is an abstract class representing a layer of features in a :cpp:class:`GDALDataset`. +- Dataset (:ref:`gdal_priv.h `): A :cpp:class:`GDALDataset` is an abstract base class representing a file or database containing one or more :cpp:class:`OGRLayer` objects. +- Drivers (:ref:`gdal_priv.h `): A :cpp:class:`GDALDriver` represents a translator for a specific format, capable of opening and possibly writing :cpp:class:`GDALDataset` objects. All available drivers are managed by the :cpp:class:`GDALDriverManager`. Geometry -------- -The geometry classes are represent various kinds of vector geometry. All the geometry classes derived from :cpp:class:`OGRGeometry` which defines the common services of all geometries. Types of geometry include :cpp:class:`OGRPoint`, :cpp:class:`OGRLineString`, :cpp:class:`OGRPolygon`, :cpp:class:`OGRGeometryCollection`, :cpp:class:`OGRMultiPolygon`, :cpp:class:`OGRMultiPoint`, and :cpp:class:`OGRMultiLineString`. +Individual geometry classes are used to represent the different types of vector geometry. All the geometry classes derive from :cpp:class:`OGRGeometry` which defines the common functionality of all geometries. Geometry types include :cpp:class:`OGRPoint`, :cpp:class:`OGRLineString`, :cpp:class:`OGRPolygon`, :cpp:class:`OGRGeometryCollection`, :cpp:class:`OGRMultiPoint`, :cpp:class:`OGRMultiLineString`, :cpp:class:`OGRMultiPolygon`, and :cpp:class:`OGRPolyhedralSurface`. +The special case of a triangular polygon can be represented as a :cpp:class:`OGRTriangle`, a non-overlapping collection of which can be represented by an :cpp:class:`OGRTriangulatedSurface`. +An additional set of types is used to store non-linear geometries: :cpp:class:`OGRCircularString`, :cpp:class:`OGRCompoundCurve`, :cpp:class:`OGRCurvePolygon`, :cpp:class:`OGRMultiCurve` and :cpp:class:`OGRMultiSurface`. -Those geometry type are extended with non-linear geometries with the :cpp:class:`OGRCircularString`, :cpp:class:`OGRCompoundCurve`, :cpp:class:`OGRCurvePolygon`, :cpp:class:`OGRMultiCurve` and :cpp:class:`OGRMultiSurface` classes. +Any of the above geometry classes can store coordinates in two (XY), three (XYZ or XYM), or four (XYZM) dimensions. -Additional intermediate abstract base classes contain functionality that could eventually be implemented by other geometry types. These include OGRCurve (base class for OGRLineString) and OGRSurface (base class for OGRPolygon). Some intermediate interfaces modeled in the simple features abstract model and SFCOM are not modeled in OGR at this time. In most cases the methods are aggregated into other classes. +Additional intermediate classes contain functionality that is used by multiple geometry types. These include :cpp:class:`OGRCurve` (base class for :cpp:class:`OGRLineString`) and :cpp:class:`OGRSurface` (base class for :cpp:class:`OGRPolygon`). Some intermediate interfaces modeled in the simple features abstract model and SFCOM are not modeled in OGR at this time. In most cases the methods are aggregated into other classes. -The :cpp:class:`OGRGeometryFactory` is used to convert well known text, and well known binary format data into geometries. These are predefined ASCII and binary formats for representing all the types of simple features geometries. +.. image:: ../../images/rfc64/classOGRGeometry_RFC64.png + :alt: Diagram of OGRGeometry and subclasses -In a manner based on the geometry object in SFCOM, the OGRGeometry includes a reference to an :cpp:class:`OGRSpatialReference` object, defining the spatial reference system of that geometry. This is normally a reference to a shared spatial reference object with reference counting for each of the OGRGeometry objects using it. +The :cpp:class:`OGRGeometryFactory` is used to convert well known text (WKT) and well known binary (WKB) format data into the appropriate :cpp:class:`OGRGeometry` subclass. These are predefined ASCII and binary formats for representing all the types of simple features geometries. -Many of the spatial analysis methods (such as computing overlaps and so forth) are not implemented at this time for OGRGeometry. +The :cpp:class:`OGRGeometry` includes a reference to an :cpp:class:`OGRSpatialReference` object, defining the spatial reference system of that geometry. This is normally a reference to a shared spatial reference object with reference counting for each of the :cpp:class:`OGRGeometry` objects using it. -While it is theoretically possible to derive other or more specific geometry classes from the existing OGRGeometry classes, this isn't an aspect that has been well thought out. In particular, it would be possible to create specialized classes using the OGRGeometryFactory without modifying it. +While it is theoretically possible to derive other or more specific geometry classes from the existing :cpp:class:`OGRGeometry` classes, this isn't an aspect that has been well thought out. In particular, it would be possible to create specialized classes using the :cpp:class:`OGRGeometryFactory` without modifying it. -Compatibility issues with non-linear geometries -++++++++++++++++++++++++++++++++++++++++++++++++ +Compatibility issues with non-linear geometries ++++++++++++++++++++++++++++++++++++++++++++++++ -Generic mechanisms have been introduced so that creating or modifying a feature with a non-linear geometry in a layer of a driver that does not support it will transform that geometry in the closest matching linear geometry. +Generic mechanisms have been introduced so that creating or modifying a feature with a non-linear geometry in a layer of a driver that does not support it will transform that geometry in the closest matching linear geometry. This linearization can be controlled using :ref:`configoptions_vector`. On the other side, when retrieving data from the OGR C API, the :cpp:func:`OGRSetNonLinearGeometriesEnabledFlag` function can be used, so that geometries and layer geometry type returned are also converted to their linear approximation if necessary. @@ -48,61 +54,98 @@ Spatial Reference The :cpp:class:`OGRSpatialReference` class is intended to store an OpenGIS Spatial Reference System definition. Currently local, geographic and projected coordinate systems are supported. Vertical coordinate systems, geocentric coordinate systems, and compound (horizontal + vertical) coordinate systems are as well supported in recent GDAL versions. -The spatial coordinate system data model is inherited from the OpenGIS Well Known Text format. A simple form of this is defined in the Simple Features specifications. A more sophisticated form is found in the Coordinate Transformation specification. The OGRSpatialReference is built on the features of the Coordinate Transformation specification but is intended to be compatible with the earlier simple features form. +The spatial coordinate system data model is inherited from the OpenGIS Well Known Text format. A simple form of this is defined in the Simple Features specifications. A more sophisticated form is found in the Coordinate Transformation specification. The :cpp:class:`OGRSpatialReference` is built on the features of the Coordinate Transformation specification but is intended to be compatible with the earlier simple features form. -There is also an associated :cpp:class:`OGRCoordinateTransformation` class that encapsulates use of PROJ for converting between different coordinate systems. There is a tutorial available describing how to use the OGRSpatialReference class. +There is also an associated :cpp:class:`OGRCoordinateTransformation` class that encapsulates use of |PROJ| for converting between different coordinate systems. Feature / Feature Definition ---------------------------- -The :cpp:class:`OGRGeometry` captures the geometry of a vector feature ... the spatial position/region of a feature. The :cpp:class:`OGRFeature` contains this geometry, and adds feature attributes, feature id, and a feature class identifier. Several geometries can be associated to a OGRFeature. +The :cpp:class:`OGRGeometry` captures the geometry of a vector feature. The :cpp:class:`OGRFeature` contains geometry, and adds feature attributes, feature id, and a feature class identifier. It may also contain styling information. Several geometries can be associated with an :cpp:class:`OGRFeature`. -The set of attributes (:cpp:class:`OGRFieldDefn`), their types, names and so forth is represented via the :cpp:class:`OGRFeatureDefn` class. One OGRFeatureDefn normally exists for a layer of features. The same definition is shared in a reference counted manner by the feature of that type (or feature class). +The set of attributes (:cpp:class:`OGRFieldDefn`), their types, names and so forth is represented via the :cpp:class:`OGRFeatureDefn` class. One :cpp:class:`OGRFeatureDefn` normally exists for a layer of features. The same definition is shared in a reference counted manner by the feature of that type (or feature class). The feature id (FID) of a feature is intended to be a unique identifier for the feature within the layer it is a member of. Freestanding features, or features not yet written to a layer may have a null (OGRNullFID) feature id. The feature ids are modeled in OGR as a 64-bit integer; however, this is not sufficiently expressive to model the natural feature ids in some formats. For instance, the GML feature id is a string. -The feature class also contains an indicator of the types of geometry allowed for that feature class (returned as an OGRwkbGeometryType from :cpp:func:`OGRFeatureDefn::GetGeomType`). If this is wkbUnknown then any type of geometry is allowed. This implies that features in a given layer can potentially be of different geometry types though they will always share a common attribute schema. +The :cpp:class:`OGRFeatureDefn` also contains an indicator of the types of geometry allowed for that feature class (returned as an :cpp:enum:`OGRwkbGeometryType` from :cpp:func:`OGRFeatureDefn::GetGeomType`). If this is :cpp:enumerator:`OGRwkbGeometryType::wkbUnknown` then any type of geometry is allowed. This implies that features in a given layer can potentially be of different geometry types though they will always share a common attribute schema. + +Several geometry fields (:cpp:class:`OGRGeomFieldDefn`) can be associated with an :cpp:class:`OGRFeatureDefn`. Each geometry field has its own indicator of geometry type allowed, returned by :cpp:func:`OGRGeomFieldDefn::GetType`, and its spatial reference system, returned by :cpp:func:`OGRGeomFieldDefn::GetSpatialRef`. + +The :cpp:class:`OGRFeatureDefn` also contains a feature class name (normally used as a layer name). + +Field Definitions +----------------- + +The behavior of each field in a feature class is defined by a shared :cpp:class:`OGRFieldDefn`. +The :cpp:class:`OGRFieldDefn` specifies the field type from the values of :cpp:enum:`OGRFieldType`. +Values stored in this field may be further restricted according to a :cpp:enum:`OGRFieldSubType`. +For example, a field may have a type of :cpp:enumerator:`OGRFieldType::OFTInteger` with a subtype of :cpp:enumerator:`OGRFieldSubType::OFSTBoolean`. + +The :cpp:class:`OGRFieldDefn` can also track whether a field is allowed to be null (:cpp:func:`OGRFieldDefn::IsNullable`), whether its value must be unique (:cpp:func:`OGRFieldDefn::IsUnique`), and formatting information such as the number of decimal digits, width, and justification. It may also define a default value in case one is not manually specified. + +Field Domains ++++++++++++++ -Several geometry fields (:cpp:class:`OGRGeomFieldDefn`) can be associated to a feature class. Each geometry field has its own indicator of geometry type allowed, returned by OGRGeomFieldDefn::GetType(), and its spatial reference system, returned by :cpp:func:`OGRGeomFieldDefn::GetSpatialRef`. +Some formats support the use of field domains that describe the values that can be stored in a given attribute field. An :cpp:class:`OGRFieldDefn` may reference a single :cpp:class:`OGRFieldDomain` that is associated with a :cpp:class:`GDALDataset`. +Programs using GDAL may use the :cpp:class:`OGRFieldDomain` to appropriately constrain user input. GDAL does not perform validation itself and will allow the storage of values that violate a field's associated :cpp:class:`OGRFieldDomain`. -The OGRFeatureDefn also contains a feature class name (normally used as a layer name). +Available types of :cpp:class:`OGRFieldDomain` include: + +- :cpp:class:`OGRCodedFieldDomain`, which constrains values those present in a specified enumeration +- :cpp:class:`OGRRangeFieldDomain`, which constrains values to a specified range +- :cpp:class:`OGRGlobFieldDomain`, which constrains values to those matching a specified pattern + +Additionally, an :cpp:class:`OGRFieldDomain` may define policies describing the values that should be assigned to domain-controlled fields when features are split or merged. Layer ----- -An :cpp:class:`OGRLayer` represents a layer of features within a data source. All features in an OGRLayer share a common schema and are of the same :cpp:class:`OGRFeatureDefn`. An OGRLayer class also contains methods for reading features from the data source. The OGRLayer can be thought of as a gateway for reading and writing features from an underlying data source, normally a file format. In SFCOM and other table based simple features implementation an OGRLayer represents a spatial table. +An :cpp:class:`OGRLayer` represents a layer of features within a data source. All features in an :cpp:class:`OGRLayer` share a common schema and are of the same :cpp:class:`OGRFeatureDefn`. An :cpp:class:`OGRLayer` class also contains methods for reading features from the data source. The :cpp:class:`OGRLayer` can be thought of as a gateway for reading and writing features from an underlying data source such as a file on disk, or the result of a database query. + +The :cpp:class:`OGRLayer` includes methods for sequential and random reading and writing. Read access (via the :cpp:func:`OGRLayer::GetNextFeature` method) normally reads all features, one at a time sequentially; however, it can be limited to return features intersecting a particular geographic region by installing a spatial filter on the :cpp:class:`OGRLayer` (via the :cpp:func:`OGRLayer::SetSpatialFilter` method). A filter on attributes can only be set with the :cpp:func:`OGRLayer::SetAttributeFilter` method. By default, all available attributes and geometries are read but this can be controlled by flagging fields as ignored (:cpp:func:`OGRLayer::SetIgnoredFields`). -The OGRLayer includes methods for sequential and random reading and writing. Read access (via the :cpp:func:`OGRLayer::GetNextFeature` method) normally reads all features, one at a time sequentially; however, it can be limited to return features intersecting a particular geographic region by installing a spatial filter on the OGRLayer (via the :cpp:func:`OGRLayer::SetSpatialFilter` method). A filter on attributes can only be set with the :cpp:func:`OGRLayer::SetAttributeFilter` method. +Starting with GDAL 3.6, as an alternative to getting features through ``GetNextFeature``, it is possible to retrieve them by batches, with a column-oriented memory layout, using the :cpp:func:`OGRLayer::GetArrowStream` method (cf :ref:`vector_api_tut_arrow_stream`). -Starting with GDAL 3.6, as an alternative to getting features through ``GetNextFeature``, it is possible to retrieve them by batches, with a column-oriented memory layout, using the :cpp:func:`OGRLayer::GetArrowStream` method (cf :ref:`vector_api_tut_arrow_stream`). +An :cpp:class:`OGRLayer` may also store an :cpp:class:`OGRStyleTable` that provides a set of styles that may be used by features in the layer. More information on GDAL's handling of feature styles can be found in the :ref:`ogr_feature_style`. -One flaw in the current OGR architecture is that the spatial and attribute filters are set directly on the OGRLayer which is intended to be the only representative of a given layer in a data source. This means it isn't possible to have multiple read operations active at one time with different spatial filters on each. +One flaw in the current OGR architecture is that the spatial and attribute filters are set directly on the :cpp:class:`OGRLayer` which is intended to be the only representative of a given layer in a data source. This means it isn't possible to have multiple read operations active at one time with different spatial filters on each. .. This aspect may be revised in the future to introduce an OGRLayerView class or something similar. -Another question that might arise is why the OGRLayer and OGRFeatureDefn classes are distinct. An OGRLayer always has a one-to-one relationship to an OGRFeatureDefn, so why not amalgamate the classes. There are two reasons: -- As defined now OGRFeature and OGRFeatureDefn don't depend on OGRLayer, so they can exist independently in memory without regard to a particular layer in a data store. +Another question that might arise is why the :cpp:class:`OGRLayer` and :cpp:class:`OGRFeatureDefn` classes are distinct. An :cpp:class:`OGRLayer` always has a one-to-one relationship to an :cpp:class:`OGRFeatureDefn`, so why not amalgamate the classes? There are two reasons: + +- As defined now :cpp:class:`OGRFeature` and :cpp:class:`OGRFeatureDefn` don't depend on :cpp:class:`OGRLayer`, so they can exist independently in memory without regard to a particular layer in a data store. - The SF CORBA model does not have a concept of a layer with a single fixed schema the way that the SFCOM and SFSQL models do. The fact that features belong to a feature collection that is potentially not directly related to their current feature grouping may be important to implementing SFCORBA support using OGR. -The OGRLayer class is an abstract base class. An implementation is expected to be subclassed for each file format driver implemented. OGRLayers are normally owned directly by their :cpp:class:`GDALDataset`, and aren't instantiated or destroyed directly. +The :cpp:class:`OGRLayer` class is an abstract base class. An implementation is expected to be subclassed for each file format driver implemented. OGRLayers are normally owned directly by their :cpp:class:`GDALDataset`, and aren't instantiated or destroyed directly. + Dataset ------- -A :cpp:class:`GDALDataset` represents a set of OGRLayer objects. This usually represents a single file, set of files, database or gateway. A GDALDataset has a list of :cpp:class:`OGRLayer` which it owns but can return references to. +A :cpp:class:`GDALDataset` represents a set of :cpp:class:`OGRLayer` objects. This usually represents a single file, set of files, database or gateway. A :cpp:class:`GDALDataset` has a list of :cpp:class:`OGRLayer` which it owns but can return references to. + +:cpp:class:`GDALDataset` is an abstract base class. An implementation is expected to be subclassed for each file format driver implemented. :cpp:class:`GDALDataset` objects are not normally instantiated directly but rather with the assistance of an :cpp:class:`GDALDriver`. Deleting an :cpp:class:`GDALDataset` closes access to the underlying persistent data source, but does not normally result in deletion of that file. + +A :cpp:class:`GDALDataset` has a name (usually a filename or database connection string) that can be used to reopen the data source with a :cpp:class:`GDALDriver`. + +The :cpp:class:`GDALDataset` also has support for executing a datasource specific command, normally a form of SQL. This is accomplished via the :cpp:func:`GDALDataset::ExecuteSQL` method. While some datasources (such as PostGIS and Oracle) pass the SQL through to an underlying database, OGR also includes support for evaluating a subset of the SQL SELECT statement against any datasource (see :ref:`ogr_sql_sqlite_dialect`.) + +When using some drivers, the :cpp:class:`GDALDataset` also offers a mechanism for to start, commit, and rollback transactions when interacting with the underlying data store. + -GDALDataset is an abstract base class. An implementation is expected to be subclassed for each file format driver implemented. GDALDataset objects are not normally instantiated directly but rather with the assistance of an :cpp:class:`GDALDriver`. Deleting an GDALDataset closes access to the underlying persistent data source, but does not normally result in deletion of that file. +A :cpp:class:`GDALDataset` may also be aware of relationships between layers (e.g., a foreign key relationship between database tables). Information about these relationships is stored in a :cpp:class:`GDALRelationshp`. -A GDALDataset has a name (usually a filename) that can be used to reopen the data source with a GDALDriver. +.. note:: -The GDALDataset also has support for executing a datasource specific command, normally a form of SQL. This is accomplished via the :cpp:func:`GDALDataset::ExecuteSQL` method. While some datasources (such as PostGIS and Oracle) pass the SQL through to an underlying database, OGR also includes support for evaluating a subset of the SQL SELECT statement against any datasource. + Earlier versions of GDAL represented vector datasets using the :cpp:class:`OGRDataSource` class. This class has been maintained for backwards compatability but is functionally equivalent to a :cpp:class:`GDALDataset` for vector data. Drivers ------- -A :cpp:class:`GDALDriver` object is instantiated for each file format supported. The GDALDriver objects are registered with the GDALDriverManager, a singleton class that is normally used to open new datasets. +A :cpp:class:`GDALDriver` object is instantiated for each file format supported. The :cpp:class:`GDALDriver` objects are registered with the :cpp:class:`GDALDriverManager`, a singleton class that is normally used to open new datasets. -It is intended that a new GDALDriver object is instantiated and define function pointers for operations like Identify(), Open() for each file format to be supported (along with a file format specific GDALDataset, and OGRLayer classes). +It is intended that a new :cpp:class:`GDALDriver` object is instantiated and define function pointers for operations like Identify(), Open() for each file format to be supported (along with a file format specific :cpp:class:`GDALDataset`, and :cpp:class:`OGRLayer` classes). -On application startup registration functions are normally called for each desired file format. These functions instantiate the appropriate GDALDriver objects, and register them with the GDALDriverManager. When a dataset is to be opened, the driver manager will normally try each GDALDataset in turn, until one succeeds, returning a GDALDataset object. +On application startup registration functions are normally called for each desired file format. These functions instantiate the appropriate :cpp:class:`GDALDriver` objects, and register them with the :cpp:class:`GDALDriverManager`. When a dataset is to be opened, the driver manager will normally try each :cpp:class:`GDALDataset` in turn, until one succeeds, returning a :cpp:class:`GDALDataset` object. From 835cefdfebe88deeb2b094c7ffa21cf14af93549 Mon Sep 17 00:00:00 2001 From: Dan Baston Date: Fri, 29 Mar 2024 09:36:48 -0400 Subject: [PATCH 04/12] Doc: Update Python API docs for FieldDefn, FieldDomain (#9579) - Correct SWIG directives so that FieldDomain docs are included - Split FieldDefn and FieldDomain docs into separate files - Remove C signatures, types - Add some examples --- doc/source/api/python/osgeo.ogr.rst | 2 +- doc/source/api/python/vector_api.rst | 4 + ogr/ogrfielddefn.cpp | 10 +- swig/include/python/docs/ogr_docs.i | 134 +++ swig/include/python/docs/ogr_fielddef_docs.i | 1000 +++-------------- .../python/docs/ogr_fielddomain_docs.i | 236 ++++ swig/include/python/ogr_python.i | 2 + swig/python/CMakeLists.txt | 2 + 8 files changed, 561 insertions(+), 829 deletions(-) create mode 100644 swig/include/python/docs/ogr_docs.i create mode 100644 swig/include/python/docs/ogr_fielddomain_docs.i diff --git a/doc/source/api/python/osgeo.ogr.rst b/doc/source/api/python/osgeo.ogr.rst index a0007287ddad..1c9972558460 100644 --- a/doc/source/api/python/osgeo.ogr.rst +++ b/doc/source/api/python/osgeo.ogr.rst @@ -5,4 +5,4 @@ osgeo.ogr module :members: :undoc-members: :show-inheritance: - :exclude-members: CreateCodedFieldDomain, CreateGeometryFromEsriJson, CreateGeometryFromGML, CreateGeometryFromJson, CreateGeometryFromWkb, CreateGeometryFromWkt, CreateGlobFieldDomain, CreateRangeFieldDomain, Feature, FeatureDefn, FieldDefn, FieldDomain, ForceTo, ForceToLineString, ForceToMultiLineString, ForceToMultiPoint, ForceToMultiPolygon, ForceToPolygon, GT_Flatten, GT_GetCollection, GT_GetCurve, GT_GetLinear, GT_HasM, GT_HasZ, GT_IsCurve, GT_IsNonLinear, GT_IsSubClassOf, GT_IsSurface, GT_SetM, GT_SetModifier, GT_SetZ, GeomFieldDefn, Geometry, GeometryTypeToName, Layer, StyleTable + :exclude-members: CreateCodedFieldDomain, CreateGeometryFromEsriJson, CreateGeometryFromGML, CreateGeometryFromJson, CreateGeometryFromWkb, CreateGeometryFromWkt, CreateGlobFieldDomain, CreateRangeFieldDomain, Feature, FeatureDefn, FieldDefn, FieldDomain, ForceTo, ForceToLineString, ForceToMultiLineString, ForceToMultiPoint, ForceToMultiPolygon, ForceToPolygon, GetFieldTypeName, GetFieldSubTypeName, GT_Flatten, GT_GetCollection, GT_GetCurve, GT_GetLinear, GT_HasM, GT_HasZ, GT_IsCurve, GT_IsNonLinear, GT_IsSubClassOf, GT_IsSurface, GT_SetM, GT_SetModifier, GT_SetZ, GeomFieldDefn, Geometry, GeometryTypeToName, Layer, StyleTable diff --git a/doc/source/api/python/vector_api.rst b/doc/source/api/python/vector_api.rst index cbd949900731..01b83dcb0dfd 100644 --- a/doc/source/api/python/vector_api.rst +++ b/doc/source/api/python/vector_api.rst @@ -105,6 +105,10 @@ FieldDefn :undoc-members: :exclude-members: thisown +.. autofunction:: osgeo.ogr.GetFieldSubTypeName + +.. autofunction:: osgeo.ogr.GetFieldTypeName + GeomFieldDefn ------------- diff --git a/ogr/ogrfielddefn.cpp b/ogr/ogrfielddefn.cpp index 1baac93980b0..6809b9e285c5 100644 --- a/ogr/ogrfielddefn.cpp +++ b/ogr/ogrfielddefn.cpp @@ -2413,7 +2413,7 @@ OGRGlobFieldDomain::OGRGlobFieldDomain(const std::string &osName, /* OGR_GlobFldDomain_Create() */ /************************************************************************/ -/** Creates a new blob field domain. +/** Creates a new glob field domain. * * This is the same as the C++ method OGRGlobFieldDomain::OGRGlobFieldDomain() * @@ -2565,12 +2565,12 @@ void OGR_FldDomain_SetSplitPolicy(OGRFieldDomainH hFieldDomain, /* OGR_FldDomain_GetMergePolicy() */ /************************************************************************/ -/** Get the split policy of the field domain. +/** Get the merge policy of the field domain. * * This is the same as the C++ method OGRFieldDomain::GetMergePolicy() * * @param hFieldDomain Field domain handle. - * @return the split policy of the field domain. + * @return the merge policy of the field domain. * @since GDAL 3.3 */ @@ -2584,12 +2584,12 @@ OGR_FldDomain_GetMergePolicy(OGRFieldDomainH hFieldDomain) /* OGR_FldDomain_SetMergePolicy() */ /************************************************************************/ -/** Set the split policy of the field domain. +/** Set the merge policy of the field domain. * * This is the same as the C++ method OGRFieldDomain::SetMergePolicy() * * @param hFieldDomain Field domain handle. - * @param policy the split policy of the field domain. + * @param policy the merge policy of the field domain. * @since GDAL 3.3 */ diff --git a/swig/include/python/docs/ogr_docs.i b/swig/include/python/docs/ogr_docs.i new file mode 100644 index 000000000000..31a8890efe41 --- /dev/null +++ b/swig/include/python/docs/ogr_docs.i @@ -0,0 +1,134 @@ +%feature("docstring") CreateCodedFieldDomain " + +Creates a new coded field domain. + +See :cpp:func:`OGRCodedFieldDomain::OGRCodedFieldDomain`. + +.. versionadded:: 3.3 + +Parameters +----------- +name : str + Domain name. Should not be ``None``. +description : str, optional + Domain description (can be ``None``) +type : int + Field type. +subtype : int + Field subtype. +enumeration : dict + Enumeration as a dictionary of (code : value) pairs. Should not be ``None``. + +Returns +-------- +FieldDomain +"; + +%feature("docstring") CreateGlobFieldDomain " + +Creates a new glob field domain. + +See :cpp:func:`OGRGlobFieldDomain::OGRGlobFieldDomain` + +.. versionadded:: 3.3 + +Parameters +----------- +name : str + Domain name. Should not be ``None``. +description : str, optional + Domain description (can be ``None``) +type : int + Field type. +subtype : int + Field subtype. +glob : str + Glob expression. Should not be ``None``. + +Returns +-------- +FieldDomain +"; + +%feature("docstring") CreateRangeFieldDomain " +Creates a new range field domain. + +See :cpp:func:`OGRRangeFieldDomain::OGRRangeFieldDomain`. + +.. versionadded:: 3.3 + +Parameters +----------- +name : str + Domain name. Should not be ``None``. +description : str, optional + Domain description (can be ``None``) +type : int + Field type. Generally numeric. Potentially :py:const:`OFTDateTime`. +subtype : int + Field subtype. +min : float, optional + Minimum value (can be ``None``). +minIsInclusive : bool + Whether the minimum value is included in the range. +max : float, optional + Maximum value (can be ``None``). +maxIsInclusive : bool + Whether the maximum value is included in the range. + +Returns +-------- +FieldDomain +"; + +// GetFieldSubTypeName +%feature("docstring") OGR_GetFieldSubTypeName " + +Fetch human readable name for a field subtype. + +See :cpp:func:`OGRFieldDefn::GetFieldSubTypeName`. + +Parameters +----------- +type : int + the field subtype to get name for. + +Returns +-------- +str + the name. + +Examples +-------- +>>> ogr.GetFieldSubTypeName(1) +'Boolean' + +>>> ogr.GetFieldSubTypeName(ogr.OFSTInt16) +'Int16' + +"; + +// GetFieldTypeName +%feature("docstring") OGR_GetFieldTypeName " +Fetch human readable name for a field type. + +See :cpp:func:`OGRFieldDefn::GetFieldTypeName`. + +Parameters +----------- +type : int + the field type code to get name for + +Returns +-------- +str + the name + +Examples +-------- +>>> ogr.GetFieldTypeName(0) +'Integer' + +>>> ogr.GetFieldTypeName(ogr.OFTReal) +'Real' +"; diff --git a/swig/include/python/docs/ogr_fielddef_docs.i b/swig/include/python/docs/ogr_fielddef_docs.i index e2dd33250fbf..abd16d969836 100644 --- a/swig/include/python/docs/ogr_fielddef_docs.i +++ b/swig/include/python/docs/ogr_fielddef_docs.i @@ -1,7 +1,10 @@ +%feature("docstring") OGRFieldDefnShadow " +Python proxy of an :cpp:class:`OGRFieldDefn`. +"; + %extend OGRFieldDefnShadow { -// File: ogrfielddefn_8cpp.xml -%feature("docstring") Create "OGRFieldDefnH OGR_Fld_Create(const -char \\*pszName, OGRFieldType eType) + +%feature("docstring") __init__ " Create a new field definition. @@ -24,80 +27,7 @@ OGRFieldDefnH: handle to the new field definition. "; -%feature("docstring") Destroy "void OGR_Fld_Destroy(OGRFieldDefnH -hDefn) - -Destroy a field definition. - -Parameters ------------ -hDefn: - handle to the field definition to destroy. -"; - -%feature("docstring") SetName "void OGR_Fld_SetName(OGRFieldDefnH -hDefn, const char \\*pszName) - -Reset the name of this field. - -This function is the same as the CPP method OGRFieldDefn::SetName(). - -Parameters ------------ -hDefn: - handle to the field definition to apply the new name to. -pszName: - the new name to apply. -"; - -%feature("docstring") GetNameRef "const char\\* -OGR_Fld_GetNameRef(OGRFieldDefnH hDefn) - -Fetch name of this field. - -This function is the same as the CPP method -OGRFieldDefn::GetNameRef(). - -Parameters ------------ -hDefn: - handle to the field definition. - -Returns --------- -str: - the name of the field definition. -"; - -%feature("docstring") SetAlternativeName "void -OGR_Fld_SetAlternativeName(OGRFieldDefnH hDefn, const char -\\*pszAlternativeName) - -Reset the alternative name (or \"alias\") for this field. - -The alternative name is an optional attribute for a field which can -provide a more user-friendly, descriptive name of a field which is not -subject to the usual naming constraints defined by the data provider. - -This is a metadata style attribute only: the alternative name cannot -be used in place of the actual field name during SQL queries or other -field name dependent API calls. - -This function is the same as the CPP method -OGRFieldDefn::SetAlternativeName(). - -.. versionadded:: 3.2 - -Parameters ------------ -hDefn: - handle to the field definition to apply the new alternative name to. -pszAlternativeName: - the new alternative name to apply. -"; - -%feature("docstring") GetAlternativeNameRef "const char\\* -OGR_Fld_GetAlternativeNameRef(OGRFieldDefnH hDefn) +%feature("docstring") GetAlternativeNameRef " Fetch the alternative name (or \"alias\") for this field. @@ -109,299 +39,115 @@ This is a metadata style attribute only: the alternative name cannot be used in place of the actual field name during SQL queries or other field name dependent API calls. -This function is the same as the CPP method -OGRFieldDefn::GetAlternativeNameRef(). +See :cpp:func:`OGRFieldDefn::GetAlternativeNameRef`. .. versionadded:: 3.2 -Parameters ------------ -hDefn: - handle to the field definition. - Returns -------- str: the alternative name of the field definition. "; -%feature("docstring") GetType "OGRFieldType -OGR_Fld_GetType(OGRFieldDefnH hDefn) - -Fetch type of this field. - -This function is the same as the CPP method OGRFieldDefn::GetType(). - -Parameters ------------ -hDefn: - handle to the field definition to get type from. - -Returns --------- -OGRFieldType: - field type. -"; - -%feature("docstring") SetType "void OGR_Fld_SetType(OGRFieldDefnH -hDefn, OGRFieldType eType) - -Set the type of this field. - -This should never be done to an OGRFieldDefn that is already part of -an OGRFeatureDefn. - -This function is the same as the CPP method OGRFieldDefn::SetType(). - -Parameters ------------ -hDefn: - handle to the field definition to set type to. -eType: - the new field type. -"; - -%feature("docstring") GetSubType "OGRFieldSubType -OGR_Fld_GetSubType(OGRFieldDefnH hDefn) - -Fetch subtype of this field. - -This function is the same as the CPP method -OGRFieldDefn::GetSubType(). - -.. versionadded:: 2.0 - -Parameters ------------ -hDefn: - handle to the field definition to get subtype from. - -Returns --------- -OGRFieldSubType: - field subtype. -"; - -%feature("docstring") SetSubType "void -OGR_Fld_SetSubType(OGRFieldDefnH hDefn, OGRFieldSubType eSubType) - -Set the subtype of this field. - -This should never be done to an OGRFieldDefn that is already part of -an OGRFeatureDefn. - -This function is the same as the CPP method -OGRFieldDefn::SetSubType(). - -.. versionadded:: 2.0 - -Parameters ------------ -hDefn: - handle to the field definition to set type to. -eSubType: - the new field subtype. -"; - -%feature("docstring") SetDefault "void -OGR_Fld_SetDefault(OGRFieldDefnH hDefn, const char \\*pszDefault) - -Set default field value. - -The default field value is taken into account by drivers (generally -those with a SQL interface) that support it at field creation time. -OGR will generally not automatically set the default field value to -null fields by itself when calling OGRFeature::CreateFeature() / -OGRFeature::SetFeature(), but will let the low-level layers to do the -job. So retrieving the feature from the layer is recommended. - -The accepted values are NULL, a numeric value, a literal value -enclosed between single quote characters (and inner single quote -characters escaped by repetition of the single quote character), -CURRENT_TIMESTAMP, CURRENT_TIME, CURRENT_DATE or a driver specific -expression (that might be ignored by other drivers). For a datetime -literal value, format should be 'YYYY/MM/DD HH:MM:SS[.sss]' -(considered as UTC time). - -Drivers that support writing DEFAULT clauses will advertise the -GDAL_DCAP_DEFAULT_FIELDS driver metadata item. - -This function is the same as the C++ method -OGRFieldDefn::SetDefault(). - -.. versionadded:: 2.0 - -Parameters ------------ -hDefn: - handle to the field definition. -pszDefault: - new default field value or NULL pointer. -"; - -%feature("docstring") GetDefault "const char\\* -OGR_Fld_GetDefault(OGRFieldDefnH hDefn) +%feature("docstring") GetDefault " Get default field value. -This function is the same as the C++ method -OGRFieldDefn::GetDefault(). - -.. versionadded:: 2.0 - -Parameters ------------ -hDefn: - handle to the field definition. +See :cpp:func:`OGRFieldDefn::GetDefault`. Returns -------- str: - default field value or NULL. + default field value or ``None``. "; -%feature("docstring") IsDefaultDriverSpecific "int -OGR_Fld_IsDefaultDriverSpecific(OGRFieldDefnH hDefn) - -Returns whether the default value is driver specific. +%feature("docstring") GetDomainName " -Driver specific default values are those that are not NULL, a numeric -value, a literal value enclosed between single quote characters, -CURRENT_TIMESTAMP, CURRENT_TIME, CURRENT_DATE or datetime literal -value. +Return the name of the field domain for this field. -This function is the same as the C++ method -OGRFieldDefn::IsDefaultDriverSpecific(). +By default an empty string is returned. -.. versionadded:: 2.0 +Field domains ( :py:class:`FieldDomain` class) are attached at the :py:class:`Dataset` level and should be retrieved with :py:meth:`Dataset.GetFieldDomain`. -Parameters ------------ -hDefn: - handle to the field definition +See :cpp:func:`OGRFieldDefn::GetDomainName`. +.. versionadded:: 3.3 Returns -------- -int: - TRUE if the default value is driver specific. +str: + the field domain name, or an empty string if there is none. "; -%feature("docstring") OGR_GetFieldTypeName "const char\\* -OGR_GetFieldTypeName(OGRFieldType eType) +%feature("docstring") GetJustify " -Fetch human readable name for a field type. +Get the justification for this field. -This function is the same as the CPP method -OGRFieldDefn::GetFieldTypeName(). - -Parameters ------------ -eType: - the field type to get name for. +See :cpp:func:`OGRFieldDefn::GetJustify`. +Note: no driver is know to use the concept of field justification. Returns -------- -str: - the name. +OGRJustification: + the justification. "; -%feature("docstring") OGR_GetFieldSubTypeName "const char\\* -OGR_GetFieldSubTypeName(OGRFieldSubType eSubType) +%feature("docstring") GetNameRef " -Fetch human readable name for a field subtype. - -This function is the same as the CPP method -OGRFieldDefn::GetFieldSubTypeName(). - -.. versionadded:: 2.0 - -Parameters ------------ -eSubType: - the field subtype to get name for. +Fetch name of this field. +See :cpp:func:`OGRFieldDefn::GetNameRef`. Returns -------- str: - the name. + the name of the field definition. "; -%feature("docstring") OGR_AreTypeSubTypeCompatible "int -OGR_AreTypeSubTypeCompatible(OGRFieldType eType, OGRFieldSubType -eSubType) +%feature("docstring") GetPrecision " -Return if type and subtype are compatible. +Get the formatting precision for this field. -.. versionadded:: 2.0 - -Parameters ------------ -eType: - the field type. -eSubType: - the field subtype. +This should normally be zero for fields of types other than :py:const:`OFTReal`. +See :cpp:func:`OGRFieldDefn::GetPrecision`. Returns -------- int: - TRUE if type and subtype are compatible + the precision. "; -%feature("docstring") GetJustify "OGRJustification -OGR_Fld_GetJustify(OGRFieldDefnH hDefn) +%feature("docstring") GetSubType " -Get the justification for this field. - -This function is the same as the CPP method -OGRFieldDefn::GetJustify(). - -Note: no driver is know to use the concept of field justification. +Fetch subtype of this field. -Parameters ------------ -hDefn: - handle to the field definition to get justification from. +See :cpp:func:`OGRFieldDefn::GetSubType`. Returns -------- -OGRJustification: - the justification. +int + field subtype code, default = :py:const:`OFSTNone` "; -%feature("docstring") SetJustify "void -OGR_Fld_SetJustify(OGRFieldDefnH hDefn, OGRJustification eJustify) - -Set the justification for this field. +%feature("docstring") GetType " -Note: no driver is know to use the concept of field justification. +Fetch type of this field. -This function is the same as the CPP method -OGRFieldDefn::SetJustify(). +See :cpp:func:`OGRFieldDefn::GetType`. -Parameters ------------ -hDefn: - handle to the field definition to set justification to. -eJustify: - the new justification. +Returns +-------- +int + field type code, e.g. :py:const:`OFTInteger` "; -%feature("docstring") GetWidth "int OGR_Fld_GetWidth(OGRFieldDefnH -hDefn) +%feature("docstring") GetWidth " Get the formatting width for this field. -This function is the same as the CPP method OGRFieldDefn::GetWidth(). - -Parameters ------------ -hDefn: - handle to the field definition to get width from. - +See :cpp:func:`OGRFieldDefn::GetWidth`. Returns -------- @@ -409,97 +155,28 @@ int: the width, zero means no specified width. "; -%feature("docstring") SetWidth "void OGR_Fld_SetWidth(OGRFieldDefnH -hDefn, int nNewWidth) - -Set the formatting width for this field in characters. - -This function is the same as the CPP method OGRFieldDefn::SetWidth(). +%feature("docstring") IsDefaultDriverSpecific " -Parameters ------------ -hDefn: - handle to the field definition to set width to. -nNewWidth: - the new width. -"; - -%feature("docstring") GetPrecision "int -OGR_Fld_GetPrecision(OGRFieldDefnH hDefn) - -Get the formatting precision for this field. - -This should normally be zero for fields of types other than OFTReal. +Returns whether the default value is driver specific. -This function is the same as the CPP method -OGRFieldDefn::GetPrecision(). +Driver specific default values are those that are not NULL, a numeric +value, a literal value enclosed between single quote characters, +CURRENT_TIMESTAMP, CURRENT_TIME, CURRENT_DATE or datetime literal +value. -Parameters ------------ -hDefn: - handle to the field definition to get precision from. +See :cpp:func:`OGRFieldDefn::IsDefaultDriverSpecific`. Returns -------- int: - the precision. -"; - -%feature("docstring") SetPrecision "void -OGR_Fld_SetPrecision(OGRFieldDefnH hDefn, int nPrecision) - -Set the formatting precision for this field in characters. - -This should normally be zero for fields of types other than OFTReal. - -This function is the same as the CPP method -OGRFieldDefn::SetPrecision(). - -Parameters ------------ -hDefn: - handle to the field definition to set precision to. -nPrecision: - the new precision. -"; - -%feature("docstring") Set "void OGR_Fld_Set(OGRFieldDefnH hDefn, -const char \\*pszNameIn, OGRFieldType eTypeIn, int nWidthIn, int -nPrecisionIn, OGRJustification eJustifyIn) - -Set defining parameters for a field in one call. - -This function is the same as the CPP method OGRFieldDefn::Set(). - -Parameters ------------ -hDefn: - handle to the field definition to set to. -pszNameIn: - the new name to assign. -eTypeIn: - the new type (one of the OFT values like OFTInteger). -nWidthIn: - the preferred formatting width. Defaults to zero indicating undefined. -nPrecisionIn: - number of decimals places for formatting, defaults to - zero indicating undefined. -eJustifyIn: - the formatting justification (OJLeft or OJRight), - defaults to OJUndefined. + TRUE if the default value is driver specific. "; -%feature("docstring") IsIgnored "int OGR_Fld_IsIgnored(OGRFieldDefnH -hDefn) +%feature("docstring") IsIgnored " Return whether this field should be omitted when fetching features. -This method is the same as the C++ method OGRFieldDefn::IsIgnored(). - -Parameters ------------ -hDefn: - handle to the field definition +See :cpp:func:`OGRFieldDefn::IsIgnored`. Returns -------- @@ -507,42 +184,18 @@ int: ignore state "; -%feature("docstring") SetIgnored "void -OGR_Fld_SetIgnored(OGRFieldDefnH hDefn, int ignore) - -Set whether this field should be omitted when fetching features. - -This method is the same as the C++ method OGRFieldDefn::SetIgnored(). - -Parameters ------------ -hDefn: - handle to the field definition -ignore: - ignore state -"; - -%feature("docstring") IsNullable "int -OGR_Fld_IsNullable(OGRFieldDefnH hDefn) +%feature("docstring") IsNullable " Return whether this field can receive null values. By default, fields are nullable. Even if this method returns FALSE (i.e not-nullable field), it doesn't -mean that OGRFeature::IsFieldSet() will necessary return TRUE, as +mean that :py:meth:`Feature.IsFieldSet` will necessary return TRUE, as fields can be temporary unset and null/not-null validation is usually -done when OGRLayer::CreateFeature()/SetFeature() is called. - -This method is the same as the C++ method OGRFieldDefn::IsNullable(). - -.. versionadded:: 2.0 - -Parameters ------------ -hDefn: - handle to the field definition +done when :py:meth:`Layer.CreateFeature`/:py:meth:`Layer.SetFeature` is called. +See :cpp:func:`OGRFieldDefn::IsNullable`. Returns -------- @@ -550,546 +203,247 @@ int: TRUE if the field is authorized to be null. "; -%feature("docstring") SetNullable "void -OGR_Fld_SetNullable(OGRFieldDefnH hDefn, int bNullableIn) - -Set whether this field can receive null values. - -By default, fields are nullable, so this method is generally called -with FALSE to set a not-null constraint. - -Drivers that support writing not-null constraint will advertise the -GDAL_DCAP_NOTNULL_FIELDS driver metadata item. - -This method is the same as the C++ method OGRFieldDefn::SetNullable(). - -.. versionadded:: 2.0 - -Parameters ------------ -hDefn: - handle to the field definition -bNullableIn: - FALSE if the field must have a not-null constraint. -"; - -%feature("docstring") IsUnique "int OGR_Fld_IsUnique(OGRFieldDefnH -hDefn) +%feature("docstring") IsUnique " Return whether this field has a unique constraint. By default, fields have no unique constraint. -This method is the same as the C++ method OGRFieldDefn::IsUnique(). +See :cpp:func:`OGRFieldDefn::IsUnique`. .. versionadded:: 3.2 -Parameters ------------ -hDefn: - handle to the field definition - - Returns -------- int: TRUE if the field has a unique constraint. "; -%feature("docstring") SetUnique "void -OGR_Fld_SetUnique(OGRFieldDefnH hDefn, int bUniqueIn) +%feature("docstring") SetAlternativeName " -Set whether this field has a unique constraint. +Reset the alternative name (or \"alias\") for this field. -By default, fields have no unique constraint, so this method is -generally called with TRUE to set a unique constraint. +The alternative name is an optional attribute for a field which can +provide a more user-friendly, descriptive name of a field which is not +subject to the usual naming constraints defined by the data provider. -Drivers that support writing unique constraint will advertise the -GDAL_DCAP_UNIQUE_FIELDS driver metadata item. field can receive null -values. +This is a metadata style attribute only: the alternative name cannot +be used in place of the actual field name during SQL queries or other +field name dependent API calls. -This method is the same as the C++ method OGRFieldDefn::SetUnique(). +See :cpp:func:`OGRFieldDefn::SetAlternativeName`. .. versionadded:: 3.2 Parameters ----------- -hDefn: - handle to the field definition -bUniqueIn: - TRUE if the field must have a unique constraint. +alternativeName : str + the new alternative name to apply. "; -%feature("docstring") GetDomainName "const char\\* -OGR_Fld_GetDomainName(OGRFieldDefnH hDefn) +%feature("docstring") SetDefault " -Return the name of the field domain for this field. +Set default field value. -By default, none (empty string) is returned. +The default field value is taken into account by drivers (generally +those with a SQL interface) that support it at field creation time. +OGR will generally not automatically set the default field value to +null fields by itself when calling OGRFeature::CreateFeature() / +OGRFeature::SetFeature(), but will let the low-level layers to do the +job. So retrieving the feature from the layer is recommended. -Field domains ( OGRFieldDomain class) are attached at the GDALDataset -level and should be retrieved with GDALDatasetGetFieldDomain(). +The accepted values are NULL, a numeric value, a literal value +enclosed between single quote characters (and inner single quote +characters escaped by repetition of the single quote character), +CURRENT_TIMESTAMP, CURRENT_TIME, CURRENT_DATE or a driver specific +expression (that might be ignored by other drivers). For a datetime +literal value, format should be 'YYYY/MM/DD HH:MM:SS[.sss]' +(considered as UTC time). -This method is the same as the C++ method -OGRFieldDefn::GetDomainName(). +Drivers that support writing DEFAULT clauses will advertise the +GDAL_DCAP_DEFAULT_FIELDS driver metadata item. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetDefault`. Parameters ----------- -hDefn: - handle to the field definition - -Returns --------- -str: - the field domain name, or an empty string if there is none. +pszValue : str + new default field value or NULL pointer. "; -%feature("docstring") SetDomainName "void -OGR_Fld_SetDomainName(OGRFieldDefnH hDefn, const char \\*pszFieldName) +%feature("docstring") SetDomainName " Set the name of the field domain for this field. -Field domains ( OGRFieldDomain) are attached at the GDALDataset level. +Field domains ( :py:class:`FieldDomain`) are attached at the :py:class:`Dataset` level. -This method is the same as the C++ method -OGRFieldDefn::SetDomainName(). +See :cpp:func:`OGRFieldDefn::SetDomainName`. .. versionadded:: 3.3 Parameters ----------- -hDefn: - handle to the field definition -pszFieldName: +name : str Field domain name. "; -%feature("docstring") OGRUpdateFieldType "void -OGRUpdateFieldType(OGRFieldDefn \\*poFDefn, OGRFieldType eNewType, -OGRFieldSubType eNewSubType) - -Update the type of a field definition by \"merging\" its existing type -with a new type. - -The update is done such as broadening the type. For example a -OFTInteger updated with OFTInteger64 will be promoted to OFTInteger64. - -.. versionadded:: 2.1 - -Parameters ------------ -poFDefn: - the field definition whose type must be updated. -eNewType: - the new field type to merge into the existing type. -eNewSubType: - the new field subtype to merge into the existing subtype. -"; - -%feature("docstring") OGR_FldDomain_Destroy "void -OGR_FldDomain_Destroy(OGRFieldDomainH hFieldDomain) - -Destroy a field domain. - -This is the same as the C++ method OGRFieldDomain::~OGRFieldDomain() - -.. versionadded:: 3.3 - -Parameters ------------ -hFieldDomain: - the field domain. -"; - -%feature("docstring") OGR_CodedFldDomain_Create "OGRFieldDomainH -OGR_CodedFldDomain_Create(const char \\*pszName, const char -\\*pszDescription, OGRFieldType eFieldType, OGRFieldSubType -eFieldSubType, const OGRCodedValue \\*enumeration) - -Creates a new coded field domain. - -This is the same as the C++ method -OGRCodedFieldDomain::OGRCodedFieldDomain() (except that the C function -copies the enumeration, whereas the C++ method moves it) - -.. versionadded:: 3.3 - -Parameters ------------ -pszName: - Domain name. Should not be NULL. -pszDescription: - Domain description (can be NULL) -eFieldType: - Field type. Generally numeric. Potentially OFTDateTime -eFieldSubType: - Field subtype. -enumeration: - Enumeration as (code, value) pairs. Should not be NULL. - The end of the enumeration is marked by a code set to NULL. The - enumeration will be copied. Each code should appear only once, but it - is the responsibility of the user to check it. - -Returns --------- -OGRFieldDomainH: - a new handle that should be freed with OGR_FldDomain_Destroy(), or - NULL in case of error. -"; - -%feature("docstring") GetUnsetField "static OGRField GetUnsetField() -"; - -%feature("docstring") OGR_RangeFldDomain_Create "OGRFieldDomainH -OGR_RangeFldDomain_Create(const char \\*pszName, const char -\\*pszDescription, OGRFieldType eFieldType, OGRFieldSubType -eFieldSubType, const OGRField \\*psMin, bool bMinIsInclusive, const -OGRField \\*psMax, bool bMaxIsInclusive) - -Creates a new range field domain. - -This is the same as the C++ method -OGRRangeFieldDomain::OGRRangeFieldDomain(). - -.. versionadded:: 3.3 - -Parameters ------------ -pszName: - Domain name. Should not be NULL. -pszDescription: - Domain description (can be NULL) -eFieldType: - Field type. Among OFTInteger, OFTInteger64, OFTReal and OFTDateTime. -eFieldSubType: - Field subtype. -psMin: - Minimum value (can be NULL). The member in the union that is - read is consistent with eFieldType -bMinIsInclusive: - Whether the minimum value is included in the range. -psMax: - Maximum value (can be NULL). The member in the union that is - read is consistent with eFieldType -bMaxIsInclusive: - Whether the maximum value is included in the range. - +%feature("docstring") SetIgnored " -Returns --------- -OGRFieldDomainH: - a new handle that should be freed with OGR_FldDomain_Destroy() -"; - -%feature("docstring") OGR_GlobFldDomain_Create "OGRFieldDomainH -OGR_GlobFldDomain_Create(const char \\*pszName, const char -\\*pszDescription, OGRFieldType eFieldType, OGRFieldSubType -eFieldSubType, const char \\*pszGlob) - -Creates a new blob field domain. - -This is the same as the C++ method -OGRGlobFieldDomain::OGRGlobFieldDomain() +Set whether this field should be omitted when fetching features. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetIgnored`. Parameters ----------- -pszName: - Domain name. Should not be NULL. -pszDescription: - Domain description (can be NULL) -eFieldType: - Field type. -eFieldSubType: - Field subtype. -pszGlob: - Glob expression. Should not be NULL. - -Returns --------- -OGRFieldDomainH: - a new handle that should be freed with OGR_FldDomain_Destroy() +bignored : bool + ignore state "; -%feature("docstring") OGR_FldDomain_GetName "const char\\* -OGR_FldDomain_GetName(OGRFieldDomainH hFieldDomain) +%feature("docstring") SetJustify " -Get the name of the field domain. +Set the justification for this field. -This is the same as the C++ method OGRFieldDomain::GetName() +Note: no driver is know to use the concept of field justification. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetJustify`. Parameters ----------- -hFieldDomain: - Field domain handle. +justify : int + the new justification -Returns +Examples -------- -str: - the field domain name. +>>> f = ogr.FieldDefn('desc', ogr.OFTString) +>>> f.SetJustify(ogr.OJRight) "; -%feature("docstring") OGR_FldDomain_GetDescription "const char\\* -OGR_FldDomain_GetDescription(OGRFieldDomainH hFieldDomain) - -Get the description of the field domain. +%feature("docstring") SetName " -This is the same as the C++ method OGRFieldDomain::GetDescription() +Reset the name of this field. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetName`. Parameters ----------- -hFieldDomain: - Field domain handle. - -Returns --------- -str: - the field domain description (might be empty string). +name : str + the new name to apply "; -%feature("docstring") OGR_FldDomain_GetDomainType "OGRFieldDomainType OGR_FldDomain_GetDomainType(OGRFieldDomainH -hFieldDomain) - -Get the type of the field domain. - -This is the same as the C++ method OGRFieldDomain::GetDomainType() - -.. versionadded:: 3.3 +%feature("docstring") SetNullable " -Parameters ------------ -hFieldDomain: - Field domain handle. - -Returns --------- -OGRFieldDomainType: - the type of the field domain. -"; - -%feature("docstring") OGR_FldDomain_GetFieldType "OGRFieldType -OGR_FldDomain_GetFieldType(OGRFieldDomainH hFieldDomain) +Set whether this field can receive null values. -Get the field type of the field domain. +By default, fields are nullable, so this method is generally called +with ``False`` to set a not-null constraint. -This is the same as the C++ method OGRFieldDomain::GetFieldType() +Drivers that support writing not-null constraint will advertise the +``GDAL_DCAP_NOTNULL_FIELDS`` driver metadata item. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetNullable`. Parameters ----------- -hFieldDomain: - Field domain handle. - -Returns --------- -OGRFieldType: - the field type of the field domain. +bNullable : bool + ``False`` if the field must have a not-null constraint. "; -%feature("docstring") OGR_FldDomain_GetFieldSubType "OGRFieldSubType -OGR_FldDomain_GetFieldSubType(OGRFieldDomainH hFieldDomain) +%feature("docstring") SetPrecision " -Get the field subtype of the field domain. +Set the formatting precision for this field in characters. -This is the same as OGRFieldDomain::GetFieldSubType() +This should normally be zero for fields of types other than :py:const:`OFTReal`. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetPrecision`. Parameters ----------- -hFieldDomain: - Field domain handle. - -Returns --------- -OGRFieldSubType: - the field subtype of the field domain. +precision : int + the new precision. "; -%feature("docstring") OGR_FldDomain_GetSplitPolicy "OGRFieldDomainSplitPolicy OGR_FldDomain_GetSplitPolicy(OGRFieldDomainH -hFieldDomain) +%feature("docstring") SetSubType " -Get the split policy of the field domain. +Set the subtype of this field. -This is the same as the C++ method OGRFieldDomain::GetSplitPolicy() +This should never be done to a :py:class:`FieldDefn` that is already part of +an :py:class:FeatureDefn`. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetSubType`. Parameters ----------- -hFieldDomain: - Field domain handle. +type : + the new field subtype. -Returns +Examples -------- -OGRFieldDomainSplitPolicy: - the split policy of the field domain. -"; - -%feature("docstring") OGR_FldDomain_SetSplitPolicy "void -OGR_FldDomain_SetSplitPolicy(OGRFieldDomainH hFieldDomain, -OGRFieldDomainSplitPolicy policy) - -Set the split policy of the field domain. - -This is the same as the C++ method OGRFieldDomain::SetSplitPolicy() - -.. versionadded:: 3.3 - -Parameters ------------ -hFieldDomain: - Field domain handle. -policy: - the split policy of the field domain. +>>> f = ogr.FieldDefn() +>>> f.SetType(ogr.OFTReal) +>>> f.SetSubType(ogr.OFSTJSON) +Warning 1: Type and subtype of field definition are not compatible. Resetting to OFSTNone +>>> f.SetSubType(ogr.OFSTFloat32) "; -%feature("docstring") OGR_FldDomain_GetMergePolicy "OGRFieldDomainMergePolicy OGR_FldDomain_GetMergePolicy(OGRFieldDomainH -hFieldDomain) +%feature("docstring") SetType " -Get the split policy of the field domain. +Set the type of this field. -This is the same as the C++ method OGRFieldDomain::GetMergePolicy() +This should never be done to a :py:class:`FieldDefn` that is already part of +an :py:class:`FeatureDefn`. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetType`. Parameters ----------- -hFieldDomain: - Field domain handle. +type : int + the new field type. -Returns +Examples -------- -OGRFieldDomainMergePolicy: - the split policy of the field domain. +>>> f = ogr.FieldDefn() +>>> f.SetType(ogr.OFTReal) "; -%feature("docstring") OGR_FldDomain_SetMergePolicy "void -OGR_FldDomain_SetMergePolicy(OGRFieldDomainH hFieldDomain, -OGRFieldDomainMergePolicy policy) - -Set the split policy of the field domain. +%feature("docstring") SetUnique " -This is the same as the C++ method OGRFieldDomain::SetMergePolicy() - -.. versionadded:: 3.3 - -Parameters ------------ -hFieldDomain: - Field domain handle. -policy: - the split policy of the field domain. -"; - -%feature("docstring") OGR_CodedFldDomain_GetEnumeration "const -OGRCodedValue\\* OGR_CodedFldDomain_GetEnumeration(OGRFieldDomainH -hFieldDomain) - -Get the enumeration as (code, value) pairs. - -The end of the enumeration is signaled by code == NULL - -This is the same as the C++ method -OGRCodedFieldDomain::GetEnumeration() - -.. versionadded:: 3.3 - -Parameters ------------ -hFieldDomain: - Field domain handle. - -Returns --------- -OGRCodedValue: - the (code, value) pairs, or nullptr in case of error. -"; - -%feature("docstring") OGR_RangeFldDomain_GetMin "const OGRField\\* -OGR_RangeFldDomain_GetMin(OGRFieldDomainH hFieldDomain, bool -\\*pbIsInclusiveOut) +Set whether this field has a unique constraint. -Get the minimum value. +By default, fields have no unique constraint, so this method is +generally called with TRUE to set a unique constraint. -Which member in the returned OGRField enum must be read depends on the -field type. +Drivers that support writing unique constraint will advertise the +``GDAL_DCAP_UNIQUE_FIELDS`` driver metadata item. -If no minimum value is set, the OGR_RawField_IsUnset() will return -true when called on the result. +Note that once a :py:class:`FieldDefn` has been added to a layer definition with +:py:meth:`Layer.AddFieldDefn`, its setter methods should not be called on the +object returned with ``GetLayerDefn().GetFieldDefn()``. Instead, +:py:meth:`Layer::AlterFieldDefn` should be called on a new instance of +:py:class:`FieldDefn`, for drivers that support :py:meth:`Layer.AlterFieldDefn`. -This is the same as the C++ method OGRRangeFieldDomain::GetMin() +See :cpp:func:`OGRFieldDefn::SetUnique`. -.. versionadded:: 3.3 +.. versionadded:: 3.2 Parameters ----------- -hFieldDomain: - Field domain handle. -pbIsInclusiveOut: - set to true if the minimum is included in the range. - -Returns --------- -OGRField: - the minimum value. +bUnique : bool + ``True`` if the field must have a unique constraint "; -%feature("docstring") OGR_RangeFldDomain_GetMax "const OGRField\\* -OGR_RangeFldDomain_GetMax(OGRFieldDomainH hFieldDomain, bool -\\*pbIsInclusiveOut) - -Get the maximum value. +%feature("docstring") SetWidth " -Which member in the returned OGRField enum must be read depends on the -field type. - -If no maximum value is set, the OGR_RawField_IsUnset() will return -true when called on the result. - -This is the same as the C++ method OGRRangeFieldDomain::GetMax() +Set the formatting width for this field in characters. -.. versionadded:: 3.3 +See :cpp:func:`OGRFieldDefn::SetWidth`. Parameters ----------- -hFieldDomain: - Field domain handle. -pbIsInclusiveOut: - set to true if the maximum is included in the range. - -Returns --------- -OGRField: - the maximum value. +width : int + the new width "; -%feature("docstring") OGR_GlobFldDomain_GetGlob "const char\\* -OGR_GlobFldDomain_GetGlob(OGRFieldDomainH hFieldDomain) - -Get the glob expression. - -This is the same as the C++ method OGRGlobFieldDomain::GetGlob() - -.. versionadded:: 3.3 +} -Parameters ------------ -hFieldDomain: - Field domain handle. -Returns --------- -str: - the glob expression, or nullptr in case of error -"; -} \ No newline at end of file diff --git a/swig/include/python/docs/ogr_fielddomain_docs.i b/swig/include/python/docs/ogr_fielddomain_docs.i new file mode 100644 index 000000000000..3190c6a94386 --- /dev/null +++ b/swig/include/python/docs/ogr_fielddomain_docs.i @@ -0,0 +1,236 @@ +%feature("docstring") OGRFieldDomainShadow " + +Python proxy of an :cpp:class:`OGRFieldDomain`. + +Created using one of: + +- :py:func:`CreateCodedFieldDomain` +- :py:func:`CreateGlobFieldDomain` +- :py:func:`CreateRangeFieldDomain` +"; + +%extend OGRFieldDomainShadow { + +%feature("docstring") GetDescription " + +Get the description of the field domain. + +See :cpp:func:`OGRFieldDomain::GetDescription`. + +.. versionadded:: 3.3 + +Returns +-------- +str + the field domain description (might be empty string). +"; + +%feature("docstring") GetName " + +Get the name of the field domain. + +See :cpp:func:`OGRFieldDomain::GetName`. + +.. versionadded:: 3.3 + +Returns +-------- +str + the field domain name. +"; + +%feature("docstring") GetDomainType " + +Get the type of the field domain. + +See :cpp:func:`OGRFieldDomain::GetDomainType`. + +.. versionadded:: 3.3 + +Returns +-------- +int + the type of the field domain. + +Examples +-------- +>>> d = ogr.CreateCodedFieldDomain('my_code', None, ogr.OFTInteger, ogr.OFSTNone, { 1 : 'owned', 2 : 'leased' }) +>>> d.GetDomainType() == ogr.OFDT_CODED +True + +"; + +%feature("docstring") GetEnumeration " + +Get the enumeration as a mapping of codes to values. + +See :cpp:func:`OGRCodedFieldDomain::GetEnumeration`. + +.. versionadded:: 3.3 + +Returns +-------- +dict + +Examples +-------- +>>> d = ogr.CreateCodedFieldDomain('my_domain', None, ogr.OFTInteger, ogr.OFSTNone, { 1 : 'owned', 2 : 'leased' }) +>>> d.GetEnumeration() +{'1': 'owned', '2': 'leased'} + +"; + +%feature("docstring") GetFieldSubType " + +Get the field subtype of the field domain. + +See :cpp:func:`OGRFieldDomain::GetFieldSubType`. + +.. versionadded:: 3.3 + +Returns +-------- +int + the field subtype of the field domain. +"; + +%feature("docstring") GetFieldType " + +Get the field type of the field domain. + +See :cpp:func:`OGRFieldDomain::GetFieldType`. + +.. versionadded:: 3.3 + +Returns +-------- +int + the field type of the field domain. +"; + +%feature("docstring") GetGlob " + +Get the glob expression. + +See :cpp:func:`OGRGlobFieldDomain::GetGlob`. + +.. versionadded:: 3.3 + +Returns +-------- +str + the glob expression, or ``None`` in case of error +"; + +%feature("docstring") GetMergePolicy " + +Get the merge policy of the field domain. + +See :cpp:func:`OGRFieldDomain::GetMergePolicy`. + +.. versionadded:: 3.3 + +Returns +-------- +int + the merge policy of the field domain (default = :py:const:`OFDMP_DEFAULT_VALUE`) +"; + +%feature("docstring") GetMaxAsDouble " + +Get the maximum value of a range domain. + +See :cpp:func:`OGRRangeFieldDomain::GetMax()` + +.. versionadded:: 3.3 + +Returns +-------- +float + the maximum value of the range +"; + +%feature("docstring") GetMaxAsString " + +Get the maximum value of a range domain. + +See :cpp:func:`OGRRangeFieldDomain::GetMax()` + +.. versionadded:: 3.3 + +Returns +-------- +str + the maximum value of the range +"; + +%feature("docstring") GetMinAsDouble " + +Get the minimum value of a range domain. + +See :cpp:func:`OGRRangeFieldDomain::GetMin()` + +.. versionadded:: 3.3 + +Returns +-------- +float + the minimum value of the range +"; + +%feature("docstring") GetMinAsString " + +Get the minimum value of a range domain. + +See :cpp:func:`OGRRangeFieldDomain::GetMin()` + +.. versionadded:: 3.3 + +Returns +-------- +str + the minimum value of the range +"; + +%feature("docstring") GetSplitPolicy " + +Get the split policy of the field domain. + +See :cpp:func:`OGRFieldDomain::GetSplitPolicy`. + +.. versionadded:: 3.3 + +Returns +-------- +int + the split policy of the field domain (default = :py:const:`OFDSP_DEFAULT_VALUE`) +"; + +%feature("docstring") SetMergePolicy " + +Set the merge policy of the field domain. + +See :cpp:func:`OGRFieldDomain::SetMergePolicy`. + +.. versionadded:: 3.3 + +Parameters +----------- +policy : int + the merge policy code of the field domain. +"; + +%feature("docstring") SetSplitPolicy " + +Set the split policy of the field domain. + +See :cpp:func:`OGRFieldDomain::SetSplitPolicy`. + +.. versionadded:: 3.3 + +policy : int + the split policy code of the field domain. +"; + +} + diff --git a/swig/include/python/ogr_python.i b/swig/include/python/ogr_python.i index bfaa702157f4..fdf545b537ce 100644 --- a/swig/include/python/ogr_python.i +++ b/swig/include/python/ogr_python.i @@ -27,6 +27,7 @@ %} */ +%include "ogr_docs.i" %include "ogr_layer_docs.i" #ifndef FROM_GDAL_I %include "ogr_datasource_docs.i" @@ -35,6 +36,7 @@ %include "ogr_feature_docs.i" %include "ogr_featuredef_docs.i" %include "ogr_fielddef_docs.i" +%include "ogr_fielddomain_docs.i" %include "ogr_geometry_docs.i" %rename (GetDriverCount) OGRGetDriverCount; diff --git a/swig/python/CMakeLists.txt b/swig/python/CMakeLists.txt index 46e4873c0f9e..9cb1b5d32db9 100644 --- a/swig/python/CMakeLists.txt +++ b/swig/python/CMakeLists.txt @@ -35,11 +35,13 @@ set(GDAL_PYTHON_CSOURCES ${PROJECT_SOURCE_DIR}/swig/include/python/docs/gdal_band_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/gdal_dataset_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/gdal_driver_docs.i + ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_datasource_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_driver_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_featuredef_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_feature_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_fielddef_docs.i + ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_fielddomain_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_geometry_docs.i ${PROJECT_SOURCE_DIR}/swig/include/python/docs/ogr_layer_docs.i) From 631da672ffde125b91917e7073afcac08b34e385 Mon Sep 17 00:00:00 2001 From: Daniel Baston Date: Fri, 29 Mar 2024 10:11:14 -0400 Subject: [PATCH 05/12] Python: Add Band.ReadAsMaskedArray --- autotest/gcore/numpy_rw.py | 32 ++++++++++++++++++++++++++++ swig/include/python/gdal_python.i | 35 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/autotest/gcore/numpy_rw.py b/autotest/gcore/numpy_rw.py index 3f8f745f9ab3..f7d82aaa7db4 100755 --- a/autotest/gcore/numpy_rw.py +++ b/autotest/gcore/numpy_rw.py @@ -983,3 +983,35 @@ def test_numpy_rw_band_read_as_array_getlasterrormsg(): with gdal.quiet_errors(): assert ds.GetRasterBand(1).ReadAsArray() is None assert gdal.GetLastErrorMsg() != "" + + +############################################################################### +# Test a band read into a masked array + + +def test_numpy_rw_masked_array_1(): + + ds = gdal.Open("data/byte.tif") + + band = ds.GetRasterBand(1) + + masked_arr = band.ReadAsMaskedArray() + + assert not numpy.any(masked_arr.mask) + + +def test_numpy_rw_masked_array_2(): + + ds = gdal.Open("data/test3_with_mask_8bit.tif") + + band = ds.GetRasterBand(1) + + arr = band.ReadAsArray() + mask = band.GetMaskBand().ReadAsArray() + + masked_arr = band.ReadAsMaskedArray() + + assert not numpy.any(masked_arr.mask[mask == 255]) + assert numpy.all(masked_arr.mask[mask != 255]) + + assert masked_arr.sum() == arr[mask == 255].sum() diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index bfe911225362..e572a47f54b5 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -552,6 +552,41 @@ void wrapper_VSIGetMemFileBuffer(const char *utf8_path, GByte **out, vsi_l_offse buf_string, buf_xsize, buf_ysize, buf_type, buf_pixel_space, buf_line_space ) + def ReadAsMaskedArray(self, xoff=0, yoff=0, win_xsize=None, win_ysize=None, + buf_xsize=None, buf_ysize=None, buf_type=None, + resample_alg=gdalconst.GRIORA_NearestNeighbour, + callback=None, + callback_data=None): + """ + Read a window of this raster band into a NumPy masked array. + + Values of the mask will be ``True`` where pixels are invalid. + + See :py:meth:`ReadAsArray` for a description of arguments. + + """ + import numpy + array = self.ReadAsArray(xoff=xoff, yoff=yoff, + win_xsize=win_xsize, win_ysize=win_ysize, + buf_xsize=buf_xsize, buf_ysize=buf_ysize, + buf_type=buf_type, + resample_alg=resample_alg, + callback=callback, callback_data=callback_data) + + if self.GetMaskFlags() != GMF_ALL_VALID: + mask = self.GetMaskBand() + mask_array = ~mask.ReadAsArray(xoff=xoff, + yoff=yoff, + win_xsize=win_xsize, + win_ysize=win_ysize, + buf_xsize=buf_xsize, + buf_ysize=buf_ysize, + resample_alg=resample_alg).astype(bool) + else: + mask_array = None + return numpy.ma.array(array, mask=mask_array) + + def ReadAsArray(self, xoff=0, yoff=0, win_xsize=None, win_ysize=None, buf_xsize=None, buf_ysize=None, buf_type=None, buf_obj=None, resample_alg=gdalconst.GRIORA_NearestNeighbour, From 2a5561694c53710b6a8dff64044c0c8fbf800fe0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 31 Mar 2024 21:29:43 +0200 Subject: [PATCH 06/12] Internal libtiff: resync with upstream --- frmts/gtiff/libtiff/tif_dirread.c | 4 +++- frmts/gtiff/libtiff/tif_dirwrite.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frmts/gtiff/libtiff/tif_dirread.c b/frmts/gtiff/libtiff/tif_dirread.c index 7c877e9be390..6fe436f63b31 100644 --- a/frmts/gtiff/libtiff/tif_dirread.c +++ b/frmts/gtiff/libtiff/tif_dirread.c @@ -4310,7 +4310,7 @@ int TIFFReadDirectory(TIFF *tif) TIFFErrorExtR( tif, module, "Failed to allocate memory for counting IFD data size at reading"); - return 0; + goto bad; } /* * Electronic Arts writes gray-scale TIFF files @@ -5279,6 +5279,8 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, TIFFErrorExtR( tif, module, "Failed to allocate memory for counting IFD data size at reading"); + if (dir) + _TIFFfreeExt(tif, dir); return 0; } diff --git a/frmts/gtiff/libtiff/tif_dirwrite.c b/frmts/gtiff/libtiff/tif_dirwrite.c index 02f7fc115193..499211aa6577 100644 --- a/frmts/gtiff/libtiff/tif_dirwrite.c +++ b/frmts/gtiff/libtiff/tif_dirwrite.c @@ -2014,7 +2014,7 @@ static int TIFFWriteDirectoryTagIfdIfd8Array(TIFF *tif, uint32_t *ndir, static void EvaluateIFDdatasizeWrite(TIFF *tif, uint32_t count, uint32_t typesize, uint32_t *ndir) { - uint64_t datalength = count * typesize; + uint64_t datalength = (uint64_t)count * typesize; if (datalength > ((tif->tif_flags & TIFF_BIGTIFF) ? 0x8U : 0x4U)) { /* LibTIFF increments write adress to an even offset, thus datalenght From dc045817ddc31e2ecf78dd127b91f563ae7a0444 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 02:59:43 +0000 Subject: [PATCH 07/12] Bump actions/setup-python from 5.0.0 to 5.1.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/0a5c61591373683505ea898e09a3ea4f39ef2b9c...82c7e631bb3cdc910f68e0081d67478d79c6982d) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/code_checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index e471b2ffd19b..3bb185c09ea1 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -82,7 +82,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 doxygen: @@ -127,7 +127,7 @@ jobs: - name: Checkout uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: 3.8 - name: Install lint tool From 9485ab4cab35ea0a4eb02610aef9577f6ebb0738 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 1 Apr 2024 20:07:35 +0200 Subject: [PATCH 08/12] GMLAS: fix crash when reading CityGML files (fixes r-spatial/sf#2371) Fixes https://github.com/r-spatial/sf/issues/2371 --- .../ogr/data/gmlas/citygml_empty_lod1.gml | 3 +++ autotest/ogr/ogr_gmlas.py | 19 +++++++++++++++++++ ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 autotest/ogr/data/gmlas/citygml_empty_lod1.gml diff --git a/autotest/ogr/data/gmlas/citygml_empty_lod1.gml b/autotest/ogr/data/gmlas/citygml_empty_lod1.gml new file mode 100644 index 000000000000..0625ccee26d3 --- /dev/null +++ b/autotest/ogr/data/gmlas/citygml_empty_lod1.gml @@ -0,0 +1,3 @@ + + + diff --git a/autotest/ogr/ogr_gmlas.py b/autotest/ogr/ogr_gmlas.py index 58f0c833f75f..6a90ccd3b281 100755 --- a/autotest/ogr/ogr_gmlas.py +++ b/autotest/ogr/ogr_gmlas.py @@ -3449,3 +3449,22 @@ def test_ogr_gmlas_get_gml_and_iso_schemas(tmp_path): ], ) assert ds + + +############################################################################### +# Test bugfix for https://github.com/r-spatial/sf/issues/2371 + + +@pytest.mark.require_curl() +def test_ogr_gmlas_bugfix_sf_2371(): + + url = ( + "http://repository.gdi-de.org/schemas/adv/citygml/building/1.0/buildingLoD1.xsd" + ) + conn = gdaltest.gdalurlopen(url, timeout=4) + if conn is None: + pytest.skip(f"cannot open {url}") + + ds = gdal.OpenEx("GMLAS:data/gmlas/citygml_empty_lod1.gml") + lyr = ds.GetLayerByName("address1") + assert lyr.GetFeatureCount() == 0 diff --git a/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp b/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp index c41959abbfb8..c8d9ec6bb38b 100644 --- a/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp +++ b/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp @@ -1488,8 +1488,8 @@ bool OGRGMLASLayer::InitReader() { CPLAssert(m_poReader == nullptr); - m_poReader = m_poDS->CreateReader(m_fpGML); m_bLayerDefnFinalized = true; + m_poReader = m_poDS->CreateReader(m_fpGML); if (m_poReader != nullptr) { m_poReader->SetLayerOfInterest(this); From c6a9903fe62199453455937cf7187fa7bdefd825 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 2 Apr 2024 00:41:19 +0200 Subject: [PATCH 09/12] typo fixes --- autotest/pyscripts/test_gdalattachpct.py | 2 +- doc/source/user/vector_data_model.rst | 2 +- frmts/gtiff/libtiff/tif_dir.h | 2 +- frmts/gtiff/libtiff/tif_dirread.c | 2 +- frmts/gtiff/libtiff/tif_dirwrite.c | 2 +- ogr/ogrsf_frmts/vfk/vfkfeature.cpp | 2 +- swig/python/gdal-utils/osgeo_utils/gdalattachpct.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/autotest/pyscripts/test_gdalattachpct.py b/autotest/pyscripts/test_gdalattachpct.py index 2d6b071a0ae9..58806f5b9bac 100755 --- a/autotest/pyscripts/test_gdalattachpct.py +++ b/autotest/pyscripts/test_gdalattachpct.py @@ -86,7 +86,7 @@ def test_gdalattachpct_basic(script_path, tmp_path, palette_file): ############################################################################### -# Test outputing to VRT +# Test outputting to VRT def test_gdalattachpct_vrt_output(script_path, tmp_path, palette_file): diff --git a/doc/source/user/vector_data_model.rst b/doc/source/user/vector_data_model.rst index e3f94cf82f0c..29dee957d410 100644 --- a/doc/source/user/vector_data_model.rst +++ b/doc/source/user/vector_data_model.rst @@ -139,7 +139,7 @@ A :cpp:class:`GDALDataset` may also be aware of relationships between layers (e. .. note:: - Earlier versions of GDAL represented vector datasets using the :cpp:class:`OGRDataSource` class. This class has been maintained for backwards compatability but is functionally equivalent to a :cpp:class:`GDALDataset` for vector data. + Earlier versions of GDAL represented vector datasets using the :cpp:class:`OGRDataSource` class. This class has been maintained for backwards compatibility but is functionally equivalent to a :cpp:class:`GDALDataset` for vector data. Drivers ------- diff --git a/frmts/gtiff/libtiff/tif_dir.h b/frmts/gtiff/libtiff/tif_dir.h index 4afb14919bae..17242eda558b 100644 --- a/frmts/gtiff/libtiff/tif_dir.h +++ b/frmts/gtiff/libtiff/tif_dir.h @@ -146,7 +146,7 @@ typedef struct td_deferstrilearraywriting; /* see TIFFDeferStrileArrayWriting() */ /* LibTIFF writes all data that does not fit into the IFD entries directly - * after the IFD tag enty part. When reading, only the IFD data directly and + * after the IFD tag entry part. When reading, only the IFD data directly and * continuously behind the IFD tags is taken into account for the IFD data * size.*/ uint64_t td_dirdatasize_write; /* auxiliary for evaluating size of IFD data diff --git a/frmts/gtiff/libtiff/tif_dirread.c b/frmts/gtiff/libtiff/tif_dirread.c index 6fe436f63b31..8f57e1b97052 100644 --- a/frmts/gtiff/libtiff/tif_dirread.c +++ b/frmts/gtiff/libtiff/tif_dirread.c @@ -4182,7 +4182,7 @@ static void CalcFinalIFDdatasizeReading(TIFF *tif, uint16_t dircount) } else { - /* Further data is no more continously after IFD */ + /* Further data is no more continuously after IFD */ break; } } diff --git a/frmts/gtiff/libtiff/tif_dirwrite.c b/frmts/gtiff/libtiff/tif_dirwrite.c index 499211aa6577..d7781fc86b10 100644 --- a/frmts/gtiff/libtiff/tif_dirwrite.c +++ b/frmts/gtiff/libtiff/tif_dirwrite.c @@ -2017,7 +2017,7 @@ static void EvaluateIFDdatasizeWrite(TIFF *tif, uint32_t count, uint64_t datalength = (uint64_t)count * typesize; if (datalength > ((tif->tif_flags & TIFF_BIGTIFF) ? 0x8U : 0x4U)) { - /* LibTIFF increments write adress to an even offset, thus datalenght + /* LibTIFF increments write address to an even offset, thus datalenght * written is also incremented. */ if (datalength & 1) datalength++; diff --git a/ogr/ogrsf_frmts/vfk/vfkfeature.cpp b/ogr/ogrsf_frmts/vfk/vfkfeature.cpp index 572ae81127b3..369d3d25b871 100644 --- a/ogr/ogrsf_frmts/vfk/vfkfeature.cpp +++ b/ogr/ogrsf_frmts/vfk/vfkfeature.cpp @@ -111,7 +111,7 @@ double IVFKFeature::GetDeterminatOfMatrixDim3(double x[3], double y[3], void IVFKFeature::GetCircleCenterFrom3Points(double c_xy[2], double x[3], double y[3]) { - /* reduce coordinates by avarage coordinate */ + /* reduce coordinates by average coordinate */ int n = 3; double sum_x = 0.0f, sum_y = 0.0f; for (int i = 0; i < n; i++) diff --git a/swig/python/gdal-utils/osgeo_utils/gdalattachpct.py b/swig/python/gdal-utils/osgeo_utils/gdalattachpct.py index ce9707550b16..80b88c11014f 100644 --- a/swig/python/gdal-utils/osgeo_utils/gdalattachpct.py +++ b/swig/python/gdal-utils/osgeo_utils/gdalattachpct.py @@ -127,7 +127,7 @@ def doit( if driver_name.upper() == "VRT": # For VRT, create the VRT first from the source dataset, so it - # correctly referes to it + # correctly refers to it out_ds = dst_driver.CreateCopy(dst_filename or "", src_ds) if out_ds is None: print(f"Cannot create {dst_filename}") From 2b85bd2635822851dc8f4d382f5b04f535ec5bb1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 6 Mar 2024 01:30:33 +0100 Subject: [PATCH 10/12] CI: cifuzz.yml: only run on OSGeo/gdal repo --- .github/workflows/cifuzz.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index c6c73bb69187..f6749919cc28 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -15,6 +15,7 @@ permissions: jobs: Fuzzing: runs-on: ubuntu-latest + if: github.repository == 'OSGeo/gdal' steps: - name: Build Fuzzers id: build From ac318b00fa3a13670d9589992eb99dd90687e92a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 22 Feb 2024 03:20:33 +0100 Subject: [PATCH 11/12] GDALDataset: make GetRasterXSize/YSize/Count/Band() const --- gcore/gdal_priv.h | 11 ++++++----- gcore/gdaldataset.cpp | 45 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/gcore/gdal_priv.h b/gcore/gdal_priv.h index cac69a93f2f4..563c02de2133 100644 --- a/gcore/gdal_priv.h +++ b/gcore/gdal_priv.h @@ -616,10 +616,11 @@ class CPL_DLL GDALDataset : public GDALMajorObject virtual CPLErr Close(); - int GetRasterXSize(); - int GetRasterYSize(); - int GetRasterCount(); + int GetRasterXSize() const; + int GetRasterYSize() const; + int GetRasterCount() const; GDALRasterBand *GetRasterBand(int); + const GDALRasterBand *GetRasterBand(int) const; /** * @brief SetQueryLoggerFunc @@ -807,8 +808,8 @@ class CPL_DLL GDALDataset : public GDALMajorObject ); #ifndef DOXYGEN_XML - void ReportError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt, ...) - CPL_PRINT_FUNC_FORMAT(4, 5); + void ReportError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt, + ...) const CPL_PRINT_FUNC_FORMAT(4, 5); static void ReportError(const char *pszDSName, CPLErr eErrClass, CPLErrorNum err_no, const char *fmt, ...) diff --git a/gcore/gdaldataset.cpp b/gcore/gdaldataset.cpp index dcfe31e3faa0..6939d827cf9e 100644 --- a/gcore/gdaldataset.cpp +++ b/gcore/gdaldataset.cpp @@ -931,7 +931,7 @@ void GDALDataset::SetBand(int nNewBand, std::unique_ptr poBand) */ -int GDALDataset::GetRasterXSize() +int GDALDataset::GetRasterXSize() const { return nRasterXSize; } @@ -968,7 +968,7 @@ int CPL_STDCALL GDALGetRasterXSize(GDALDatasetH hDataset) */ -int GDALDataset::GetRasterYSize() +int GDALDataset::GetRasterYSize() const { return nRasterYSize; } @@ -1028,6 +1028,43 @@ GDALRasterBand *GDALDataset::GetRasterBand(int nBandId) return nullptr; } +/************************************************************************/ +/* GetRasterBand() */ +/************************************************************************/ + +/** + + \brief Fetch a band object for a dataset. + + See GetBands() for a C++ iterator version of this method. + + Equivalent of the C function GDALGetRasterBand(). + + @param nBandId the index number of the band to fetch, from 1 to + GetRasterCount(). + + @return the nBandId th band object + +*/ + +const GDALRasterBand *GDALDataset::GetRasterBand(int nBandId) const + +{ + if (papoBands) + { + if (nBandId < 1 || nBandId > nBands) + { + ReportError(CE_Failure, CPLE_IllegalArg, + "GDALDataset::GetRasterBand(%d) - Illegal band #\n", + nBandId); + return nullptr; + } + + return papoBands[nBandId - 1]; + } + return nullptr; +} + /************************************************************************/ /* GDALGetRasterBand() */ /************************************************************************/ @@ -1058,7 +1095,7 @@ GDALRasterBandH CPL_STDCALL GDALGetRasterBand(GDALDatasetH hDS, int nBandId) * @return the number of raster bands. */ -int GDALDataset::GetRasterCount() +int GDALDataset::GetRasterCount() const { return papoBands ? nBands : 0; } @@ -4500,7 +4537,7 @@ int GDALDataset::CloseDependentDatasets() */ void GDALDataset::ReportError(CPLErr eErrClass, CPLErrorNum err_no, - const char *fmt, ...) + const char *fmt, ...) const { va_list args; va_start(args, fmt); From 7f4a54ba833baa5cb4a7c05308bef28be2752066 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 5 Mar 2024 23:31:40 +0100 Subject: [PATCH 12/12] VRT: add a new mode to apply chained processing steps that apply to several bands at the same time The following built-in algorithms are introduced, and typically applied in the following order: - Dehazing: remove haze effects by applying (subsampled) gain and offset auxiliary datasets. - BandAffineCombination: to perform an affine transformation combination of bands. - Trimming: local thresholding of saturation - LUT: apply a look-up table (band per band) --- autotest/gdrivers/vrtprocesseddataset.py | 1258 +++++++++++++ doc/source/drivers/raster/vrt.rst | 16 + .../drivers/raster/vrt_processed_dataset.rst | 261 +++ frmts/vrt/CMakeLists.txt | 2 + frmts/vrt/data/gdalvrt.xsd | 136 +- frmts/vrt/vrtdataset.cpp | 170 +- frmts/vrt/vrtdataset.h | 182 +- frmts/vrt/vrtdriver.cpp | 14 +- frmts/vrt/vrtprocesseddataset.cpp | 1342 ++++++++++++++ frmts/vrt/vrtprocesseddatasetfunctions.cpp | 1579 +++++++++++++++++ frmts/vrt/vrtsources.cpp | 92 +- gcore/gdal.h | 109 ++ 12 files changed, 5008 insertions(+), 153 deletions(-) create mode 100755 autotest/gdrivers/vrtprocesseddataset.py create mode 100644 doc/source/drivers/raster/vrt_processed_dataset.rst create mode 100644 frmts/vrt/vrtprocesseddataset.cpp create mode 100644 frmts/vrt/vrtprocesseddatasetfunctions.cpp diff --git a/autotest/gdrivers/vrtprocesseddataset.py b/autotest/gdrivers/vrtprocesseddataset.py new file mode 100755 index 000000000000..7ebec7613d0d --- /dev/null +++ b/autotest/gdrivers/vrtprocesseddataset.py @@ -0,0 +1,1258 @@ +#!/usr/bin/env pytest +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Test VRTProcessedDataset support. +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import struct + +import gdaltest +import pytest + +from osgeo import gdal + +############################################################################### +# Test error cases in general VRTProcessedDataset XML structure + + +def test_vrtprocesseddataset_errors(tmp_vsimem): + + with pytest.raises(Exception, match="Input element missing"): + gdal.Open( + """ + + """ + ) + + with pytest.raises( + Exception, + match="Input element should have a SourceFilename or VRTDataset element", + ): + gdal.Open( + """ + + + """ + ) + + with pytest.raises(Exception): # "No such file or directory'", but O/S dependent + gdal.Open( + """ + + + """ + ) + + with pytest.raises( + Exception, + match="Missing one of rasterXSize, rasterYSize or bands on VRTDataset", + ): + gdal.Open( + """ + + + """ + ) + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 10, 5, 3) + src_ds.GetRasterBand(1).Fill(1) + src_ds.GetRasterBand(2).Fill(2) + src_ds.GetRasterBand(3).Fill(3) + src_ds.Close() + + with pytest.raises(Exception, match="ProcessingSteps element missing"): + gdal.Open( + f""" + + {src_filename} + + + """ + ) + + with pytest.raises( + Exception, match="Inconsistent declared VRT dimensions with input dataset" + ): + gdal.Open( + f""" + + {src_filename} + + + """ + ) + + with pytest.raises( + Exception, match="Inconsistent declared VRT dimensions with input dataset" + ): + gdal.Open( + f""" + + {src_filename} + + + """ + ) + + with pytest.raises(Exception, match="At least one step should be defined"): + gdal.Open( + f""" + + {src_filename} + + + + """ + ) + + +############################################################################### +# Test nominal cases of BandAffineCombination algorithm + + +def test_vrtprocesseddataset_affine_combination_nominal(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 3) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x03") + src_ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x02\x06") + src_ds.GetRasterBand(3).WriteRaster(0, 0, 2, 1, b"\x03\x03") + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 10,0,1,0 + 20,0,0,1 + 30,1,0,0 + 15 + 32 + + + + """ + ) + assert ds.RasterXSize == 2 + assert ds.RasterYSize == 1 + assert ds.RasterCount == 3 + assert ds.GetSpatialRef() is None + assert ds.GetGeoTransform(can_return_null=True) is None + assert ds.GetRasterBand(1).DataType == gdal.GDT_Byte + assert struct.unpack("B" * 2, ds.GetRasterBand(1).ReadRaster()) == (15, 10 + 6) + assert struct.unpack("B" * 2, ds.GetRasterBand(2).ReadRaster()) == (20 + 3, 20 + 3) + assert struct.unpack("B" * 2, ds.GetRasterBand(3).ReadRaster()) == (30 + 1, 32) + + +############################################################################### +# Test several steps in a VRTProcessedDataset + + +def test_vrtprocesseddataset_several_steps(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 10, 5, 3) + src_ds.GetRasterBand(1).Fill(1) + src_ds.GetRasterBand(2).Fill(2) + src_ds.GetRasterBand(3).Fill(3) + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 0,0,1,0 + 0,0,0,1 + 0,1,0,0 + + + BandAffineCombination + 0,0,1,0 + 0,0,0,1 + 0,1,0,0 + + + BandAffineCombination + 0,0,1,0 + 0,0,0,1 + 0,1,0,0 + + + + """ + ) + assert ds.RasterXSize == 10 + assert ds.RasterYSize == 5 + assert ds.RasterCount == 3 + assert ds.GetSpatialRef() is None + assert ds.GetGeoTransform(can_return_null=True) is None + assert ds.GetRasterBand(1).DataType == gdal.GDT_Byte + assert ds.GetRasterBand(1).ComputeRasterMinMax(False) == (1, 1) + assert ds.GetRasterBand(2).ComputeRasterMinMax(False) == (2, 2) + assert ds.GetRasterBand(3).ComputeRasterMinMax(False) == (3, 3) + + +############################################################################### +# Test nominal cases of BandAffineCombination algorithm with nodata + + +def test_vrtprocesseddataset_affine_combination_nodata(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + src_ds.GetRasterBand(1).SetNoDataValue(1) + src_ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x03\x03") + src_ds.GetRasterBand(2).SetNoDataValue(1) + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 0,1,1 + 0,1,-1 + + + + """ + ) + assert ds.GetRasterBand(1).DataType == gdal.GDT_Byte + assert struct.unpack("B" * 2, ds.GetRasterBand(1).ReadRaster()) == (1, 5) + # 0 should actually be 3-2=1, but this is the nodata value hence the replacement value + assert struct.unpack("B" * 2, ds.GetRasterBand(2).ReadRaster()) == (1, 0) + + +def test_vrtprocesseddataset_affine_combination_nodata_as_parameter(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + src_ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x03\x03") + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 0,1,1 + 256,1,-1 + 1 + 255 + Byte + + + + """ + ) + assert ds.GetRasterBand(1).DataType == gdal.GDT_Byte + assert struct.unpack("B" * 2, ds.GetRasterBand(1).ReadRaster()) == (255, 5) + # 254 should actually be 256+1*2+(-1)*3=255, but this is the nodata value hence the replacement value + assert struct.unpack("B" * 2, ds.GetRasterBand(2).ReadRaster()) == (255, 254) + + +############################################################################### +# Test replacement_nodata logic of BandAffineCombination + + +def test_vrtprocesseddataset_affine_combination_replacement_nodata(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + src_ds.GetRasterBand(2).WriteRaster(0, 0, 2, 1, b"\x03\x03") + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 0,1,1 + 256,1,-1 + 1 + 255 + 128 + + + + """ + ) + assert ds.GetRasterBand(1).DataType == gdal.GDT_Byte + assert struct.unpack("B" * 2, ds.GetRasterBand(1).ReadRaster()) == (255, 5) + # 254 should actually be 256+1*2+(-1)*3=255, but this is the nodata value hence the replacement value + assert struct.unpack("B" * 2, ds.GetRasterBand(2).ReadRaster()) == (255, 128) + + +############################################################################### +# Test error cases of BandAffineCombination algorithm + + +def test_vrtprocesseddataset_affine_combination_errors(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 10, 5, 3) + src_ds.GetRasterBand(1).Fill(1) + src_ds.GetRasterBand(2).Fill(2) + src_ds.GetRasterBand(3).Fill(3) + src_ds.Close() + + with pytest.raises( + Exception, + match="Step 'Affine combination of band values' lacks required Argument 'coefficients_{band}'", + ): + gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + + + + """ + ) + + with pytest.raises( + Exception, match="Argument coefficients_1 has 3 values, whereas 4 are expected" + ): + gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 10,0,1 + + + + """ + ) + + with pytest.raises(Exception, match="Argument coefficients_3 is missing"): + gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 10,0,1,0 + 10,0,1,0 + 10,0,1,0 + + + + """ + ) + + with pytest.raises( + Exception, + match="Final step expect 3 bands, but only 1 coefficient_XX are provided", + ): + gdal.Open( + f""" + + {src_filename} + + + + BandAffineCombination + 10,0,1,0 + + + + """ + ) + + +############################################################################### +# Test nominal cases of LUT algorithm + + +def test_vrtprocesseddataset_lut_nominal(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 3, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x01\x02\x03") + src_ds.GetRasterBand(2).WriteRaster(0, 0, 3, 1, b"\x01\x02\x03") + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + LUT + 1.5:10,2.5:20 + 1.5:100,2.5:200 + + + + """ + ) + assert struct.unpack("B" * 3, ds.GetRasterBand(1).ReadRaster()) == (10, 15, 20) + assert struct.unpack("B" * 3, ds.GetRasterBand(2).ReadRaster()) == (100, 150, 200) + + +############################################################################### +# Test nominal cases of LUT algorithm with nodata coming from input dataset + + +def test_vrtprocesseddataset_lut_nodata(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 4, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 4, 1, b"\x00\x01\x02\x03") + src_ds.GetRasterBand(1).SetNoDataValue(0) + src_ds.GetRasterBand(2).WriteRaster(0, 0, 4, 1, b"\x00\x01\x02\x03") + src_ds.GetRasterBand(2).SetNoDataValue(0) + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + LUT + 1.5:10,2.5:20 + 1.5:100,2.5:200 + + + + """ + ) + assert struct.unpack("B" * 4, ds.GetRasterBand(1).ReadRaster()) == (0, 10, 15, 20) + assert struct.unpack("B" * 4, ds.GetRasterBand(2).ReadRaster()) == ( + 0, + 100, + 150, + 200, + ) + + +############################################################################### +# Test nominal cases of LUT algorithm with nodata set as a parameter + + +def test_vrtprocesseddataset_lut_nodata_as_parameter(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 4, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 4, 1, b"\x00\x01\x02\x03") + src_ds.GetRasterBand(2).WriteRaster(0, 0, 4, 1, b"\x00\x01\x02\x03") + src_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + LUT + 1.5:10,2.5:20 + 1.5:100,2.5:200 + 0 + 1 + + + + """ + ) + assert struct.unpack("B" * 4, ds.GetRasterBand(1).ReadRaster()) == (1, 10, 15, 20) + assert struct.unpack("B" * 4, ds.GetRasterBand(2).ReadRaster()) == ( + 1, + 100, + 150, + 200, + ) + + +############################################################################### +# Test error cases of LUT algorithm + + +def test_vrtprocesseddataset_lut_errors(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 3, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x01\x02\x03") + src_ds.GetRasterBand(2).WriteRaster(0, 0, 3, 1, b"\x01\x02\x03") + src_ds.Close() + + with pytest.raises(Exception, match="Step 'nr 1' lacks required Argument"): + gdal.Open( + f""" + + {src_filename} + + + + LUT + + + + """ + ) + + with pytest.raises(Exception, match="Invalid value for argument 'lut_1'"): + gdal.Open( + f""" + + {src_filename} + + + + LUT + 1.5:10,2.5 + + + + """ + ) + + with pytest.raises(Exception, match="Invalid band in argument 'lut_3'"): + gdal.Open( + f""" + + {src_filename} + + + + LUT + 1.5:10,2.5:20 + 1.5:10,2.5:20 + + + + """ + ) + + with pytest.raises(Exception, match="Missing lut_XX element"): + gdal.Open( + f""" + + {src_filename} + + + + LUT + 1.5:10,2.5:20 + + + + """ + ) + + +############################################################################### +# Test nominal case of Dehazing algorithm + + +def test_vrtprocesseddataset_dehazing_nominal(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 6, 1, 2) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 6, 1, b"\x01\x02\x03\xff\x01\x01") + src_ds.GetRasterBand(2).WriteRaster(0, 0, 6, 1, b"\x01\x02\x03\xff\x01\x01") + src_ds.GetRasterBand(1).SetNoDataValue(255) + src_ds.GetRasterBand(2).SetNoDataValue(255) + src_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + src_ds.Close() + + gain_filename = str(tmp_vsimem / "gain.tif") + gain_ds = gdal.GetDriverByName("GTiff").Create(gain_filename, 6, 1, 2) + gain_ds.GetRasterBand(1).WriteRaster(0, 0, 6, 1, b"\x02\x04\x06\x01\xfe\x01") + gain_ds.GetRasterBand(2).WriteRaster(0, 0, 6, 1, b"\x03\x05\x07\x01\xfe\x01") + gain_ds.GetRasterBand(1).SetNoDataValue(254) + gain_ds.GetRasterBand(2).SetNoDataValue(254) + gain_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + gain_ds.Close() + + offset_filename = str(tmp_vsimem / "offset.tif") + offset_ds = gdal.GetDriverByName("GTiff").Create(offset_filename, 6, 1, 2) + offset_ds.GetRasterBand(1).WriteRaster(0, 0, 6, 1, b"\x01\x02\x03\x01\x01\xfd") + offset_ds.GetRasterBand(2).WriteRaster(0, 0, 6, 1, b"\x02\x03\x04\x01\x01\xfd") + offset_ds.GetRasterBand(1).SetNoDataValue(253) + offset_ds.GetRasterBand(2).SetNoDataValue(253) + offset_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + offset_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {gain_filename} + 1 + {gain_filename} + 2 + {offset_filename} + 1 + {offset_filename} + 2 + 2 + 16 + + + + """ + ) + assert struct.unpack("B" * 6, ds.GetRasterBand(1).ReadRaster()) == ( + 2, + 6, + 15, + 255, + 255, + 255, + ) + assert struct.unpack("B" * 6, ds.GetRasterBand(2).ReadRaster()) == ( + 2, + 7, + 16, + 255, + 255, + 255, + ) + + +############################################################################### +# Test nominal case of Dehazing algorithm where gain and offset have a lower +# resolution than the input dataset + + +def test_vrtprocesseddataset_dehazing_different_resolution(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 6, 2, 1) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 6, 2, b"\x01\x01\x02\x02\x03\x03" * 2) + src_ds.SetGeoTransform([0, 0.5, 0, 0, 0, 0.5]) + src_ds.Close() + + gain_filename = str(tmp_vsimem / "gain.tif") + gain_ds = gdal.GetDriverByName("GTiff").Create(gain_filename, 3, 1, 1) + gain_ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x02\x04\x06") + gain_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + gain_ds.Close() + + offset_filename = str(tmp_vsimem / "offset.tif") + offset_ds = gdal.GetDriverByName("GTiff").Create(offset_filename, 3, 1, 1) + offset_ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x01\x02\x03") + offset_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + offset_ds.Close() + + ds = gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {gain_filename} + 1 + {offset_filename} + 1 + + + + """ + ) + assert struct.unpack("B" * 12, ds.GetRasterBand(1).ReadRaster()) == ( + 1, + 2, + 6, + 8, + 15, + 15, + 1, + 2, + 6, + 8, + 15, + 15, + ) + + +############################################################################### +# Test error cases of Dehazing algorithm + + +def test_vrtprocesseddataset_dehazing_error(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 3, 1, 1) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 3, 1, b"\x01\x02\x03") + src_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + src_ds.Close() + + with pytest.raises( + Exception, + match="Step 'nr 1' lacks required Argument 'offset_dataset_band_{band}'", + ): + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {src_filename} + 1 + + + + """ + ) + + with pytest.raises( + Exception, + match="Invalid band in argument 'gain_dataset_filename_2'", + ): + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {src_filename} + 1 + {src_filename} + 1 + + + + """ + ) + + with pytest.raises( + Exception, + match="Invalid band in argument 'gain_dataset_band_2'", + ): + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {src_filename} + 1 + {src_filename} + 1 + + + + """ + ) + + with pytest.raises( + Exception, + match="Invalid band in argument 'offset_dataset_filename_2'", + ): + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {src_filename} + 1 + {src_filename} + 1 + + + + """ + ) + + with pytest.raises( + Exception, + match="Invalid band in argument 'offset_dataset_band_2'", + ): + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {src_filename} + 1 + {src_filename} + 1 + + + + """ + ) + + with pytest.raises( + Exception, + match=r"Invalid band number \(2\) for a gain dataset", + ): + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {src_filename} + 2 + {src_filename} + 1 + + + + """ + ) + + with pytest.raises(Exception): # "No such file or directory'", but O/S dependent + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + invalid + 1 + {src_filename} + 1 + + + + """ + ) + + nogt_filename = str(tmp_vsimem / "nogt.tif") + ds = gdal.GetDriverByName("GTiff").Create(nogt_filename, 1, 1, 1) + ds.Close() + + with pytest.raises(Exception, match="lacks a geotransform"): + gdal.Open( + f""" + + {src_filename} + + + + Dehazing + {nogt_filename} + 1 + {nogt_filename} + 1 + + + + """ + ) + + +############################################################################### +# Test nominal cases of Trimming algorithm + + +def test_vrtprocesseddataset_trimming_nominal(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 6, 1, 4) + + R = 100.0 + G = 150.0 + B = 200.0 + NIR = 100.0 + + src_ds.GetRasterBand(1).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, int(R), 150, 200, 0, 0, 0) + ) + src_ds.GetRasterBand(2).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, int(G), 200, 100, 0, 0, 0) + ) + src_ds.GetRasterBand(3).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, int(B), 100, 150, 0, 0, 0) + ) + src_ds.GetRasterBand(4).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, int(NIR), 150, 200, 0, 0, 0) + ) + src_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + src_ds.Close() + + trimming_filename = str(tmp_vsimem / "trimming.tif") + trimming_ds = gdal.GetDriverByName("GTiff").Create(trimming_filename, 6, 1, 1) + + localMaxRGB = 205.0 + + trimming_ds.GetRasterBand(1).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, int(localMaxRGB), 210, 220, 0, 0, 0) + ) + trimming_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + trimming_ds.Close() + + top_rgb = 200.0 + tone_ceil = 190.0 + top_margin = 0.1 + + ds = gdal.Open( + f""" + + {src_filename} + + + + Trimming + {trimming_filename} + {top_rgb} + {tone_ceil} + {top_margin} + + + + """ + ) + + # Do algorithm at hand + + # Extract local saturation value from trimming image + reducedRGB = min((1.0 - top_margin) * top_rgb / localMaxRGB, 1) + + # RGB bands specific process + maxRGB = max(R, G, B) + toneMaxRGB = min(tone_ceil / maxRGB, 1) + toneR = min(tone_ceil / R, 1) + toneG = min(tone_ceil / G, 1) + toneB = min(tone_ceil / B, 1) + outputR = min(reducedRGB * R * toneR / toneMaxRGB, top_rgb) + outputG = min(reducedRGB * G * toneG / toneMaxRGB, top_rgb) + outputB = min(reducedRGB * B * toneB / toneMaxRGB, top_rgb) + + # Other bands processing (NIR, ...): only apply RGB reduction factor + outputNIR = reducedRGB * NIR + + # print(outputR, outputG, outputB, outputNIR) + + assert ( + round(outputR) + == struct.unpack("B", ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1))[0] + ) + assert ( + round(outputG) + == struct.unpack("B", ds.GetRasterBand(2).ReadRaster(0, 0, 1, 1))[0] + ) + assert ( + round(outputB) + == struct.unpack("B", ds.GetRasterBand(3).ReadRaster(0, 0, 1, 1))[0] + ) + assert ( + round(outputNIR) + == struct.unpack("B", ds.GetRasterBand(4).ReadRaster(0, 0, 1, 1))[0] + ) + + assert struct.unpack("B" * 6, ds.GetRasterBand(1).ReadRaster()) == ( + 92, # round(outputR) + 135, + 164, + 0, + 0, + 0, + ) + assert struct.unpack("B" * 6, ds.GetRasterBand(2).ReadRaster()) == ( + 139, # round(outputG) + 171, + 86, + 0, + 0, + 0, + ) + assert struct.unpack("B" * 6, ds.GetRasterBand(3).ReadRaster()) == ( + 176, # round(outputB) + 90, + 129, + 0, + 0, + 0, + ) + assert struct.unpack("B" * 6, ds.GetRasterBand(4).ReadRaster()) == ( + 88, # round(outputNIR) + 129, + 164, + 0, + 0, + 0, + ) + + +############################################################################### +# Test error cases of Trimming algorithm + + +def test_vrtprocesseddataset_trimming_errors(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 6, 1, 4) + src_ds.GetRasterBand(1).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, 100, 150, 200, 0, 0, 0) + ) + src_ds.GetRasterBand(2).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, 150, 200, 100, 0, 0, 0) + ) + src_ds.GetRasterBand(3).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, 200, 100, 150, 0, 0, 0) + ) + src_ds.GetRasterBand(4).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, 100, 150, 200, 0, 0, 0) + ) + src_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + src_ds.Close() + + trimming_filename = str(tmp_vsimem / "trimming.tif") + trimming_ds = gdal.GetDriverByName("GTiff").Create(trimming_filename, 6, 1, 1) + trimming_ds.GetRasterBand(1).WriteRaster( + 0, 0, 6, 1, struct.pack("B" * 6, 200, 210, 220, 0, 0, 0) + ) + trimming_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + trimming_ds.Close() + + trimming_two_bands_filename = str(tmp_vsimem / "trimming_two_bands.tif") + trimming_ds = gdal.GetDriverByName("GTiff").Create( + trimming_two_bands_filename, 6, 1, 2 + ) + trimming_ds.SetGeoTransform([0, 1, 0, 0, 0, 1]) + trimming_ds.Close() + + with pytest.raises(Exception): + gdal.Open( + f""" + + {src_filename} + + + + Trimming + invalid + 200 + 190 + 0.1 + + + + """ + ) + + for val in (0, 5): + with pytest.raises(Exception, match="Invalid band in argument 'red_band'"): + gdal.Open( + f""" + + {src_filename} + + + + Trimming + {trimming_filename} + {val} + 200 + 190 + 0.1 + + + + """ + ) + + for val in (0, 5): + with pytest.raises(Exception, match="Invalid band in argument 'green_band'"): + gdal.Open( + f""" + + {src_filename} + + + + Trimming + {trimming_filename} + {val} + 200 + 190 + 0.1 + + + + """ + ) + + for val in (0, 5): + with pytest.raises(Exception, match="Invalid band in argument 'blue_band'"): + gdal.Open( + f""" + + {src_filename} + + + + Trimming + {trimming_filename} + {val} + 200 + 190 + 0.1 + + + + """ + ) + + for (red_band, green_band, blue_band) in [(1, 1, 3), (3, 2, 3), (1, 3, 3)]: + with pytest.raises( + Exception, + match="red_band, green_band and blue_band must have distinct values", + ): + gdal.Open( + f""" + + {src_filename} + + + + Trimming + {trimming_filename} + {red_band} + {green_band} + {blue_band} + 200 + 190 + 0.1 + + + + """ + ) + + with pytest.raises(Exception, match="Trimming dataset should have a single band"): + gdal.Open( + f""" + + {src_filename} + + + + Trimming + {trimming_two_bands_filename} + 200 + 190 + 0.1 + + + + """ + ) + + +############################################################################### +# Test that serialization (for example due to statistics computation) properly +# works + + +def test_vrtprocesseddataset_serialize(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create(src_filename, 2, 1, 1) + src_ds.GetRasterBand(1).WriteRaster(0, 0, 2, 1, b"\x01\x02") + src_ds.Close() + + vrt_filename = str(tmp_vsimem / "the.vrt") + content = f""" + + + {src_filename} + + + + BandAffineCombination + 10,1 + + + + """ + with gdaltest.tempfile(vrt_filename, content): + ds = gdal.Open(vrt_filename) + assert struct.unpack("B" * 2, ds.GetRasterBand(1).ReadRaster()) == (11, 12) + assert ds.GetRasterBand(1).GetStatistics(False, False) == [0.0, 0.0, 0.0, -1.0] + ds.GetRasterBand(1).ComputeStatistics(False) + ds.Close() + + ds = gdal.Open(vrt_filename) + assert struct.unpack("B" * 2, ds.GetRasterBand(1).ReadRaster()) == (11, 12) + assert ds.GetRasterBand(1).GetStatistics(False, False) == [ + 11.0, + 12.0, + 11.5, + 0.5, + ] diff --git a/doc/source/drivers/raster/vrt.rst b/doc/source/drivers/raster/vrt.rst index 1c860c439ef2..4aa8da0f06a9 100644 --- a/doc/source/drivers/raster/vrt.rst +++ b/doc/source/drivers/raster/vrt.rst @@ -1907,6 +1907,22 @@ See the dedicated :ref:`vrt_multidimensional` page. vrt_multidimensional +Processed dataset VRT +--------------------- + +.. versionadded:: 3.9 + +A VRT processed dataset is a specific variant of the :ref:`raster.vrt` format, +to apply chained processing steps that apply to several bands at the same time. + +See the dedicated :ref:`vrt_processed_dataset` page. + +.. toctree:: + :maxdepth: 1 + :hidden: + + vrt_processed_dataset + vrt:// connection string ------------------------ diff --git a/doc/source/drivers/raster/vrt_processed_dataset.rst b/doc/source/drivers/raster/vrt_processed_dataset.rst new file mode 100644 index 000000000000..9f00f7f13d74 --- /dev/null +++ b/doc/source/drivers/raster/vrt_processed_dataset.rst @@ -0,0 +1,261 @@ +.. _vrt_processed_dataset: + +================================================================================ +VRT processed dataset +================================================================================ + +.. versionadded:: 3.9 + +A VRT processed dataset is a specific variant of the :ref:`raster.vrt` format, +to apply chained processing steps that apply to several bands at the same time. + +The following built-in algorithms are introduced, and may typically be applied +in the following order: + +- Dehazing: remove haze effects by applying (subsampled) gain and offset + auxiliary datasets. + +- BandAffineCombination: perform an affine transformation combination of bands. + +- Trimming: apply local thresholding of saturation + +- LUT: apply a look-up table (band per band) + +More algorithms can be registered are run-time with the :cpp:func:`GDALVRTRegisterProcessedDatasetFunc` +function` + +Here's an example of such a file to apply various correction to a R,G,B,NIR dataset: + +.. code-block:: xml + + + + source.tif + + + + + Dehazing + + true + + gains.tif + gains.tif + gains.tif + gains.tif + 1 + 2 + 3 + 4 + + offsets.tif + offsets.tif + offsets.tif + offsets.tif + 1 + 2 + 3 + 4 + + 0 + 1 + 10000 + + + + BandAffineCombination + 0,1.2,-0.2,0.0,0.0 + 0,-0.03,1.03,0.0,0.0 + 0,0.0,0.0,1.0,0.0 + 0,0.0,0.0,0.0,1.0 + + 1 + 10000 + + + + Trimming + true + trimming.tif + 10000 + 0 + 10000 + + + + LUT + + 0:0,10000.0:255 + + + 0:0,10000.0:255 + + + 0:0,10000.0:255 + + + 0:0,10000.0:255 + + + + + + Red + + + Green + + + Blue + + + + + +.vrt format +----------- + +The ``VRTDataset`` root element must have a ``subClass="VRTProcessedDataset"`` attribute. + +The following child elements of ``VRTDataset`` may be defined: ``SRS``, ``GeoTransform``, ``Metadata``. If they are not explicitly set, they are inferred from the input dataset. + +``VRTRasterBand`` elements may be explicitly defined, in particular if the data type of the virtual dataset after all processing steps is different from the input one, or if the number of output bands is different from the number of input bands. If there is no explicit ``VRTRasterBand`` element, the number and data types of input bands are used implicitly. When explicitly defined, ``VRTRasterBand`` elements must have a ``subClass="VRTProcessedRasterBand"`` attribute. +` +It must also have the 2 following child elements: + +- ``Input``, which must have one and only one of the following ``SourceFilename`` or ``VRTDataset`` as child elements, to define the input dataset to which to apply the processing steps. + +- ``ProcessingSteps``, with at least one child ``Step`` element. + +Each ``Step`` must have a ``Algorithm`` child element, and an optional ``name`` attribute. +The value of ``Algorithm`` must be a registered VRTProcessedDataset function. At time of writing, the following 4 algorithms are defined: ``Dehazing``, ``BandAffineCombination``, ``Trimming`` and ``LUT``. + +A ``Step`` will generally have one or several ``Argument`` child elements, some of them being required, others optional. Consult the documentation of each algorithm. + +Dehazing algorithm +------------------ + +Remove haze effects by applying (subsampled) gain and offset auxiliary datasets. + +The gain and offset auxiliary datasets must have a georeferencing consistent of +the input dataset, but may have a different resolution. + +The formula applied by that algorithm is: ``output_value = clamp(input_value * gain - offset, min, max)`` + +The following required arguments must be specified: + +- ``gain_dataset_filename_{band}``: Filename to the gain dataset, where {band} must be replaced by 1 to the number of input bands. + +- ``gain_dataset_band_{band}``: Band number corresponding to ``gain_dataset_filename_{band}``, where {band} must be replaced by 1 to the number of input bands. + +- ``offset_dataset_filename_{band}``: Filename to the offset dataset, where {band} must be replaced by 1 to the number of input bands. + +- ``offset_dataset_band_{band}``: Band number corresponding to ``offset_dataset_filename_{band}``, where {band} must be replaced by 1 to the number of input bands. + + +The following optional arguments may be specified: + +- ``relative_filenames``: Whether gain and offset filenames are relative to the VRT. Allowed values are ``true`` and ``false``. Defaults to ``false`` + +- ``min``: Clamp minimum value, applied before writing the output value. + +- ``max``: Clamp maximum value, applied before writing the output value. + +- ``nodata``: Override the input nodata value coming from the previous step (or the input dataset for the first step). + +- ``gain_nodata``: Override the nodata value coming from the gain dataset(s). + +- ``offset_nodata``: Override the nodata value coming from the offset dataset(s). + + +BandAffineCombination algorithm +------------------------------- + +Perform an affine transformation combination of bands. + +The following required argument must be specified: + +- ``coefficients_{band}``: Comma-separated coefficients for combining bands where {band} must be replaced by 1 to the number of output bands. The number of coefficients in each argument must be 1 + number_of_input_bands, where the first coefficient is a constant, the second coefficient is the weight of the first input band, the third coefficient is the weight of the second input band, etc. + + +The following optional arguments may be specified: + +- ``src_nodata``: Override the input nodata value coming from the previous step (or the input dataset for the first step). + +- ``dst_nodata``: Set the output nodata value. + +- ``replacement_nodata``: Value to substitute to a valid computed value that would be equal to dst_nodata. + +- ``dst_intended_datatype``: Intended datatype of output (which might be different than the working data type). Used to infer an appropriate value for replacement_nodata when it is not specified. + +- ``min``: Clamp minimum value, applied before writing the output value. + +- ``max``: Clamp maximum value, applied before writing the output value. + + +Trimming algorithm +------------------ + +Apply local thresholding of saturation, with a special processing of the R,G,B bands compared to other bands. + +The pseudo algorithm used for each pixel is: + +.. code-block:: + + // Extract local saturation value from trimming image + localMaxRGB = value from TrimmingImage + reducedRGB = min ( (1-top_margin)*top_rgb/localMaxRGB ; 1) + + // RGB bands specific process + RGB[] = get red, green, blue components of input buffer + maxRGB = max(RGB[]) + toneMaxRGB = min ( toneCeil/maxRGB ; 1) + toneBand[] = min ( toneCeil/RGB[] ; 1) + + output_value_RGB[] = min ( reducedRGB*RGB[]*toneBand[] / toneMaxRGB ; topRGB) + + // Other bands processing (NIR, ...): only apply RGB reduction factor + Trimmed(OtherBands[]) = reducedRGB * OtherBands[] + + +The following required arguments must be specified: + +- ``trimming_dataset_filename``: Filename of the trimming dataset. It must have one single band. It must have a georeferencing consistent of the input dataset, but may have a different resolution. + +- ``top_rgb``: Maximum saturating RGB output value. + +- ``tone_ceil``: Maximum threshold beyond which we give up saturation. + +- ``top_margin``: Margin to allow for dynamics in brighest areas (between 0 and 1, should be close to 0) + + +The following optional arguments may be specified: + +- ``relative_filenames``: Whether the trimming dataset filename is relative to the VRT. Allowed values are ``true`` and ``false``. Defaults to ``false`` + +- ``red_band``: Index (one-based) of the red band. Defaults to 1. + +- ``green_band``: Index (one-based) of the green band. Defaults to 1. + +- ``blue_band``: Index (one-based) of the blue band. Defaults to 1. + +- ``nodata``: Override the input nodata value coming from the previous step (or the input dataset for the first step). + +- ``trimming_nodata``: Override the nodata value coming from the trimming dataset. + + +LUT +--- + +Apply a look-up table (band per band), typically to get from UInt16 to Byte data types. + +The following required argument must be specified: + +- ``lut_{band}``: List of the form ``[src value 1]:[dest value 1],[src value 2]:[dest value 2],....``. {band} must be replaced by 1 to the number of bands. + + +The following optional arguments may be specified: + +- ``src_nodata``: Override the input nodata value coming from the previous step (or the input dataset for the first step). + +- ``dst_nodata``: Set the output nodata value. diff --git a/frmts/vrt/CMakeLists.txt b/frmts/vrt/CMakeLists.txt index ba770561f569..447d48eaedbe 100644 --- a/frmts/vrt/CMakeLists.txt +++ b/frmts/vrt/CMakeLists.txt @@ -14,6 +14,8 @@ add_gdal_driver( vrtdataset.cpp pixelfunctions.cpp vrtpansharpened.cpp + vrtprocesseddataset.cpp + vrtprocesseddatasetfunctions.cpp vrtmultidim.cpp gdaltileindexdataset.cpp STRONG_CXX_WFLAGS) diff --git a/frmts/vrt/data/gdalvrt.xsd b/frmts/vrt/data/gdalvrt.xsd index 5e204df9ab98..89ce1496a4b7 100644 --- a/frmts/vrt/data/gdalvrt.xsd +++ b/frmts/vrt/data/gdalvrt.xsd @@ -30,30 +30,76 @@ ****************************************************************************/ --> - - - - - - - - - - - - - - - - - - - - - - + + + Root element + + + + + + + + + + + + May be repeated + + + + + May be repeated + + + + + + Allowed only if subClass="VRTWarpedDataset" + + + + + Allowed only if subClass="VRTPansharpenedDataset" + + + + + Allowed only if subClass="VRTProcessedDataset" + + + + + Allowed only if subClass="VRTProcessedDataset" + + + + + only for multidimensional dataset + + + + + + + + + + + + + + + + + Added in GDAL 3.9 + + + + + @@ -158,6 +204,51 @@ + + + + + + + + + + + + + + + + + + Processing step of a VRTPansharpenedDataset + + + + + Builtin allowed names are BandAffineCombination, LUT, Dehazing, Trimming. More algorithms can be registered at run-time. + + + + + + + + + + Argument of a processing function + + + + + + Allowed names are specific of each processing function + + + + + + @@ -234,6 +325,7 @@ + diff --git a/frmts/vrt/vrtdataset.cpp b/frmts/vrt/vrtdataset.cpp index b620daa227d4..7e9758ae1ff2 100644 --- a/frmts/vrt/vrtdataset.cpp +++ b/frmts/vrt/vrtdataset.cpp @@ -97,10 +97,6 @@ VRTDataset::~VRTDataset() { VRTDataset::FlushCache(true); - if (m_poSRS) - m_poSRS->Release(); - if (m_poGCP_SRS) - m_poGCP_SRS->Release(); CPLFree(m_pszVRTPath); delete m_poMaskBand; @@ -150,6 +146,17 @@ CPLErr VRTPansharpenedDataset::FlushCache(bool bAtClosing) /* FlushCache() */ /************************************************************************/ +CPLErr VRTProcessedDataset::FlushCache(bool bAtClosing) + +{ + return VRTFlushCacheStruct::FlushCache(*this, + bAtClosing); +} + +/************************************************************************/ +/* FlushCache() */ +/************************************************************************/ + template CPLErr VRTFlushCacheStruct::FlushCache(T &obj, bool bAtClosing) { @@ -312,7 +319,7 @@ CPLXMLNode *VRTDataset::SerializeToXML(const char *pszVRTPathIn) /* -------------------------------------------------------------------- */ if (!m_asGCPs.empty()) { - GDALSerializeGCPListToXML(psDSTree, m_asGCPs, m_poGCP_SRS); + GDALSerializeGCPListToXML(psDSTree, m_asGCPs, m_poGCP_SRS.get()); } /* -------------------------------------------------------------------- */ @@ -405,10 +412,18 @@ CPLXMLNode *CPL_STDCALL VRTSerializeToXML(VRTDatasetH hDataset, /************************************************************************/ VRTRasterBand *VRTDataset::InitBand(const char *pszSubclass, int nBand, - bool bAllowPansharpened) + bool bAllowPansharpenedOrProcessed) { VRTRasterBand *poBand = nullptr; - if (EQUAL(pszSubclass, "VRTSourcedRasterBand")) + if (auto poProcessedDS = dynamic_cast(this)) + { + if (bAllowPansharpenedOrProcessed && + EQUAL(pszSubclass, "VRTProcessedRasterBand")) + { + poBand = new VRTProcessedRasterBand(poProcessedDS, nBand); + } + } + else if (EQUAL(pszSubclass, "VRTSourcedRasterBand")) poBand = new VRTSourcedRasterBand(this, nBand); else if (EQUAL(pszSubclass, "VRTDerivedRasterBand")) poBand = new VRTDerivedRasterBand(this, nBand); @@ -417,13 +432,17 @@ VRTRasterBand *VRTDataset::InitBand(const char *pszSubclass, int nBand, else if (EQUAL(pszSubclass, "VRTWarpedRasterBand") && dynamic_cast(this) != nullptr) poBand = new VRTWarpedRasterBand(this, nBand); - else if (bAllowPansharpened && + else if (bAllowPansharpenedOrProcessed && EQUAL(pszSubclass, "VRTPansharpenedRasterBand") && dynamic_cast(this) != nullptr) poBand = new VRTPansharpenedRasterBand(this, nBand); - else + + if (!poBand) + { CPLError(CE_Failure, CPLE_AppDefined, "VRTRasterBand of unrecognized subclass '%s'.", pszSubclass); + } + return poBand; } @@ -443,9 +462,7 @@ CPLErr VRTDataset::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPathIn) const CPLXMLNode *psSRSNode = CPLGetXMLNode(psTree, "SRS"); if (psSRSNode) { - if (m_poSRS) - m_poSRS->Release(); - m_poSRS = new OGRSpatialReference(); + m_poSRS.reset(new OGRSpatialReference()); m_poSRS->SetFromUserInput( CPLGetXMLValue(psSRSNode, nullptr, ""), OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get()); @@ -500,7 +517,9 @@ CPLErr VRTDataset::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPathIn) /* -------------------------------------------------------------------- */ if (const CPLXMLNode *psGCPList = CPLGetXMLNode(psTree, "GCPList")) { - GDALDeserializeGCPListFromXML(psGCPList, m_asGCPs, &m_poGCP_SRS); + OGRSpatialReference *poSRS = nullptr; + GDALDeserializeGCPListFromXML(psGCPList, m_asGCPs, &poSRS); + m_poGCP_SRS.reset(poSRS); } /* -------------------------------------------------------------------- */ @@ -557,6 +576,13 @@ CPLErr VRTDataset::XMLInit(const CPLXMLNode *psTree, const char *pszVRTPathIn) { const char *pszSubclass = CPLGetXMLValue(psChild, "subclass", "VRTSourcedRasterBand"); + if (dynamic_cast(this) && + !EQUAL(pszSubclass, "VRTProcessedRasterBand")) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Only subClass=VRTProcessedRasterBand supported"); + return CE_Failure; + } VRTRasterBand *poBand = InitBand(pszSubclass, l_nBands + 1, true); if (poBand != nullptr && @@ -636,10 +662,7 @@ CPLErr VRTDataset::SetGCPs(int nGCPCountIn, const GDAL_GCP *pasGCPListIn, const OGRSpatialReference *poGCP_SRS) { - if (m_poGCP_SRS) - m_poGCP_SRS->Release(); - - m_poGCP_SRS = poGCP_SRS ? poGCP_SRS->Clone() : nullptr; + m_poGCP_SRS.reset(poGCP_SRS ? poGCP_SRS->Clone() : nullptr); m_asGCPs = gdal::GCP::fromC(pasGCPListIn, nGCPCountIn); SetNeedsFlush(); @@ -654,12 +677,7 @@ CPLErr VRTDataset::SetGCPs(int nGCPCountIn, const GDAL_GCP *pasGCPListIn, CPLErr VRTDataset::SetSpatialRef(const OGRSpatialReference *poSRS) { - if (m_poSRS) - m_poSRS->Release(); - if (poSRS) - m_poSRS = poSRS->Clone(); - else - m_poSRS = nullptr; + m_poSRS.reset(poSRS ? poSRS->Clone() : nullptr); SetNeedsFlush(); @@ -860,8 +878,7 @@ GDALDataset *VRTDataset::Open(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ /* Turn the XML representation into a VRTDataset. */ /* -------------------------------------------------------------------- */ - VRTDataset *poDS = static_cast( - OpenXML(pszXML, pszVRTPath, poOpenInfo->eAccess)); + VRTDataset *poDS = OpenXML(pszXML, pszVRTPath, poOpenInfo->eAccess); if (poDS != nullptr) poDS->m_bNeedsFlush = false; @@ -1495,8 +1512,8 @@ GDALDataset *VRTDataset::OpenVRTProtocol(const char *pszSpec) /* of the dataset. */ /************************************************************************/ -GDALDataset *VRTDataset::OpenXML(const char *pszXML, const char *pszVRTPath, - GDALAccess eAccessIn) +VRTDataset *VRTDataset::OpenXML(const char *pszXML, const char *pszVRTPath, + GDALAccess eAccessIn) { /* -------------------------------------------------------------------- */ @@ -1517,8 +1534,10 @@ GDALDataset *VRTDataset::OpenXML(const char *pszXML, const char *pszVRTPath, const bool bIsPansharpened = strcmp(pszSubClass, "VRTPansharpenedDataset") == 0; + const bool bIsProcessed = strcmp(pszSubClass, "VRTProcessedDataset") == 0; - if (!bIsPansharpened && CPLGetXMLNode(psRoot, "Group") == nullptr && + if (!bIsPansharpened && !bIsProcessed && + CPLGetXMLNode(psRoot, "Group") == nullptr && (CPLGetXMLNode(psRoot, "rasterXSize") == nullptr || CPLGetXMLNode(psRoot, "rasterYSize") == nullptr || CPLGetXMLNode(psRoot, "VRTRasterBand") == nullptr)) @@ -1535,7 +1554,8 @@ GDALDataset *VRTDataset::OpenXML(const char *pszXML, const char *pszVRTPath, const int nXSize = atoi(CPLGetXMLValue(psRoot, "rasterXSize", "0")); const int nYSize = atoi(CPLGetXMLValue(psRoot, "rasterYSize", "0")); - if (!bIsPansharpened && CPLGetXMLNode(psRoot, "VRTRasterBand") != nullptr && + if (!bIsPansharpened && !bIsProcessed && + CPLGetXMLNode(psRoot, "VRTRasterBand") != nullptr && !GDALCheckDatasetDimensions(nXSize, nYSize)) { return nullptr; @@ -1546,6 +1566,8 @@ GDALDataset *VRTDataset::OpenXML(const char *pszXML, const char *pszVRTPath, poDS = new VRTWarpedDataset(nXSize, nYSize); else if (bIsPansharpened) poDS = new VRTPansharpenedDataset(nXSize, nYSize); + else if (bIsProcessed) + poDS = new VRTProcessedDataset(nXSize, nYSize); else { poDS = new VRTDataset(nXSize, nYSize); @@ -2781,4 +2803,94 @@ void VRTDataset::ClearStatistics() GDALDataset::ClearStatistics(); } +/************************************************************************/ +/* BuildSourceFilename() */ +/************************************************************************/ + +/* static */ +std::string VRTDataset::BuildSourceFilename(const char *pszFilename, + const char *pszVRTPath, + bool bRelativeToVRT) +{ + std::string osSrcDSName; + if (pszVRTPath != nullptr && bRelativeToVRT) + { + // Try subdatasetinfo API first + // Note: this will become the only branch when subdatasetinfo will become + // available for NITF_IM, RASTERLITE and TILEDB + const auto oSubDSInfo{GDALGetSubdatasetInfo(pszFilename)}; + if (oSubDSInfo && !oSubDSInfo->GetPathComponent().empty()) + { + auto path{oSubDSInfo->GetPathComponent()}; + osSrcDSName = oSubDSInfo->ModifyPathComponent( + CPLProjectRelativeFilename(pszVRTPath, path.c_str())); + GDALDestroySubdatasetInfo(oSubDSInfo); + } + else + { + bool bDone = false; + for (const char *pszSyntax : VRTDataset::apszSpecialSyntax) + { + CPLString osPrefix(pszSyntax); + osPrefix.resize(strchr(pszSyntax, ':') - pszSyntax + 1); + if (pszSyntax[osPrefix.size()] == '"') + osPrefix += '"'; + if (EQUALN(pszFilename, osPrefix, osPrefix.size())) + { + if (STARTS_WITH_CI(pszSyntax + osPrefix.size(), "{ANY}")) + { + const char *pszLastPart = strrchr(pszFilename, ':') + 1; + // CSV:z:/foo.xyz + if ((pszLastPart[0] == '/' || pszLastPart[0] == '\\') && + pszLastPart - pszFilename >= 3 && + pszLastPart[-3] == ':') + { + pszLastPart -= 2; + } + CPLString osPrefixFilename = pszFilename; + osPrefixFilename.resize(pszLastPart - pszFilename); + osSrcDSName = + osPrefixFilename + + CPLProjectRelativeFilename(pszVRTPath, pszLastPart); + bDone = true; + } + else if (STARTS_WITH_CI(pszSyntax + osPrefix.size(), + "{FILENAME}")) + { + CPLString osFilename(pszFilename + osPrefix.size()); + size_t nPos = 0; + if (osFilename.size() >= 3 && osFilename[1] == ':' && + (osFilename[2] == '\\' || osFilename[2] == '/')) + nPos = 2; + nPos = osFilename.find( + pszSyntax[osPrefix.size() + strlen("{FILENAME}")], + nPos); + if (nPos != std::string::npos) + { + const CPLString osSuffix = osFilename.substr(nPos); + osFilename.resize(nPos); + osSrcDSName = osPrefix + + CPLProjectRelativeFilename( + pszVRTPath, osFilename) + + osSuffix; + bDone = true; + } + } + break; + } + } + if (!bDone) + { + osSrcDSName = + CPLProjectRelativeFilename(pszVRTPath, pszFilename); + } + } + } + else + { + osSrcDSName = pszFilename; + } + return osSrcDSName; +} + /*! @endcond */ diff --git a/frmts/vrt/vrtdataset.h b/frmts/vrt/vrtdataset.h index 7ef902a63b47..f9df8ac59f47 100644 --- a/frmts/vrt/vrtdataset.h +++ b/frmts/vrt/vrtdataset.h @@ -47,6 +47,7 @@ #include CPLErr GDALRegisterDefaultPixelFunc(); +void GDALVRTRegisterDefaultProcessedDatasetFuncs(); CPLString VRTSerializeNoData(double dfVal, GDALDataType eDataType, int nPrecision); @@ -202,6 +203,7 @@ template struct VRTFlushCacheStruct class VRTWarpedDataset; class VRTPansharpenedDataset; +class VRTProcessedDataset; class VRTGroup; class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset @@ -210,16 +212,14 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset friend struct VRTFlushCacheStruct; friend struct VRTFlushCacheStruct; friend struct VRTFlushCacheStruct; + friend struct VRTFlushCacheStruct; friend class VRTSourcedRasterBand; + friend class VRTSimpleSource; friend VRTDatasetH CPL_STDCALL VRTCreate(int nXSize, int nYSize); - OGRSpatialReference *m_poSRS = nullptr; - - int m_bGeoTransformSet = false; - double m_adfGeoTransform[6]; - std::vector m_asGCPs{}; - OGRSpatialReference *m_poGCP_SRS = nullptr; + std::unique_ptr + m_poGCP_SRS{}; bool m_bNeedsFlush = false; bool m_bWritable = true; @@ -247,8 +247,13 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset VRTSource::WorkingState m_oWorkingState{}; + static constexpr const char *const apszSpecialSyntax[] = { + "NITF_IM:{ANY}:{FILENAME}", "PDF:{ANY}:{FILENAME}", + "RASTERLITE:{FILENAME},{ANY}", "TILEDB:\"{FILENAME}\":{ANY}", + "TILEDB:{FILENAME}:{ANY}"}; + VRTRasterBand *InitBand(const char *pszSubclass, int nBand, - bool bAllowPansharpened); + bool bAllowPansharpenedOrProcessed); static GDALDataset *OpenVRTProtocol(const char *pszSpec); bool AddVirtualOverview(int nOvFactor, const char *pszResampling); @@ -263,6 +268,11 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset int m_nBlockXSize = 0; int m_nBlockYSize = 0; + std::unique_ptr m_poSRS{}; + + int m_bGeoTransformSet = false; + double m_adfGeoTransform[6]; + virtual int CloseDependentDatasets() override; public: @@ -287,7 +297,7 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset const OGRSpatialReference *GetSpatialRef() const override { - return m_poSRS; + return m_poSRS.get(); } CPLErr SetSpatialRef(const OGRSpatialReference *poSRS) override; @@ -306,7 +316,7 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset const OGRSpatialReference *GetGCPSpatialRef() const override { - return m_poGCP_SRS; + return m_poGCP_SRS.get(); } virtual const GDAL_GCP *GetGCPs() override; @@ -375,8 +385,8 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset static int Identify(GDALOpenInfo *); static GDALDataset *Open(GDALOpenInfo *); - static GDALDataset *OpenXML(const char *, const char * = nullptr, - GDALAccess eAccess = GA_ReadOnly); + static VRTDataset *OpenXML(const char *, const char * = nullptr, + GDALAccess eAccess = GA_ReadOnly); static GDALDataset *Create(const char *pszName, int nXSize, int nYSize, int nBands, GDALDataType eType, char **papszOptions); @@ -385,6 +395,10 @@ class CPL_DLL VRTDataset CPL_NON_FINAL : public GDALDataset CSLConstList papszRootGroupOptions, CSLConstList papszOptions); static CPLErr Delete(const char *pszFilename); + + static std::string BuildSourceFilename(const char *pszFilename, + const char *pszVRTPath, + bool bRelativeToVRT); }; /************************************************************************/ @@ -526,6 +540,132 @@ class VRTPansharpenedDataset final : public VRTDataset } }; +/************************************************************************/ +/* VRTPansharpenedDataset */ +/************************************************************************/ + +/** Specialized implementation of VRTDataset that chains several processing + * steps applied on all bands at a time. + * + * @since 3.9 + */ +class VRTProcessedDataset final : public VRTDataset +{ + public: + VRTProcessedDataset(int nXSize, int nYSize); + ~VRTProcessedDataset() override; + + virtual CPLErr FlushCache(bool bAtClosing) override; + + virtual CPLErr XMLInit(const CPLXMLNode *, const char *) override; + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override; + + void GetBlockSize(int *, int *) const; + + // GByte whose initialization constructor does nothing +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif + struct NoInitByte + { + GByte value; + + // cppcheck-suppress uninitMemberVar + NoInitByte() + { + // do nothing + /* coverity[uninit_member] */ + } + + inline operator GByte() const + { + return value; + } + }; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + private: + friend class VRTProcessedRasterBand; + + //! Data for a processing step. + struct Step + { + //! Algorithm name + std::string osAlgorithm{}; + + //! Arguments to pass to the processing function. + CPLStringList aosArguments{}; + + //! Data type of the input buffer. + GDALDataType eInDT = GDT_Unknown; + + //! Data type of the output buffer. + GDALDataType eOutDT = GDT_Unknown; + + //! Number of input bands. + int nInBands = 0; + + //! Number of output bands. + int nOutBands = 0; + + //! Nodata values (nInBands) of the input bands. + std::vector adfInNoData{}; + + //! Nodata values (nOutBands) of the output bands. + std::vector adfOutNoData{}; + + //! Working data structure (private data of the implementation of the function) + VRTPDWorkingDataPtr pWorkingData = nullptr; + + // NOTE: if adding a new member, edit the move constructor and + // assignment operators! + + Step() = default; + ~Step(); + Step(Step &&); + Step &operator=(Step &&); + + private: + Step(const Step &) = delete; + Step &operator=(const Step &) = delete; + void deinit(); + }; + + //! Directory of the VRT + std::string m_osVRTPath{}; + + //! Source dataset + std::unique_ptr m_poSrcDS{}; + + //! Processing steps. + std::vector m_aoSteps{}; + + //! Backup XML tree passed to XMLInit() + CPLXMLTreeCloser m_oXMLTree{nullptr}; + + //! Overview datasets (dynamically generated from the ones of m_poSrcDS) + std::vector> m_apoOverviewDatasets{}; + + //! Input buffer of a processing step + std::vector m_abyInput{}; + + //! Output buffer of a processing step + std::vector m_abyOutput{}; + + CPLErr Init(const CPLXMLNode *, const char *, + const VRTProcessedDataset *poParentDS, + GDALDataset *poParentSrcDS, int iOvrLevel); + + bool ParseStep(const CPLXMLNode *psStep, bool bIsFinalStep, + GDALDataType &eCurrentDT, int &nCurrentBandCount, + std::vector &adfInNoData, + std::vector &adfOutNoData); + bool ProcessRegion(int nXOff, int nYOff, int nBufXSize, int nBufYSize); +}; + /************************************************************************/ /* VRTRasterBand */ /* */ @@ -895,6 +1035,26 @@ class VRTPansharpenedRasterBand final : public VRTRasterBand } }; +/************************************************************************/ +/* VRTProcessedRasterBand */ +/************************************************************************/ + +class VRTProcessedRasterBand final : public VRTRasterBand +{ + public: + VRTProcessedRasterBand(VRTProcessedDataset *poDS, int nBand, + GDALDataType eDataType = GDT_Unknown); + + virtual CPLErr IReadBlock(int, int, void *) override; + + virtual int GetOverviewCount() override; + virtual GDALRasterBand *GetOverview(int) override; + + virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) override; +}; + /************************************************************************/ /* VRTDerivedRasterBand */ /************************************************************************/ diff --git a/frmts/vrt/vrtdriver.cpp b/frmts/vrt/vrtdriver.cpp index 2883944b4e13..5e69096de56a 100644 --- a/frmts/vrt/vrtdriver.cpp +++ b/frmts/vrt/vrtdriver.cpp @@ -34,6 +34,8 @@ #include "gdal_alg_priv.h" #include "gdal_frmts.h" +#include + /*! @cond Doxygen_Suppress */ /************************************************************************/ @@ -504,8 +506,16 @@ void GDALRegister_VRT() if (GDALGetDriverByName("VRT") != nullptr) return; - // First register the pixel functions - GDALRegisterDefaultPixelFunc(); + static std::once_flag flag; + std::call_once(flag, + []() + { + // First register the pixel functions + GDALRegisterDefaultPixelFunc(); + + // Register functions for VRTProcessedDataset + GDALVRTRegisterDefaultProcessedDatasetFuncs(); + }); VRTDriver *poDriver = new VRTDriver(); diff --git a/frmts/vrt/vrtprocesseddataset.cpp b/frmts/vrt/vrtprocesseddataset.cpp new file mode 100644 index 000000000000..c7b80f803a38 --- /dev/null +++ b/frmts/vrt/vrtprocesseddataset.cpp @@ -0,0 +1,1342 @@ +/****************************************************************************** + * + * Project: Virtual GDAL Datasets + * Purpose: Implementation of VRTProcessedDataset. + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "cpl_minixml.h" +#include "cpl_string.h" +#include "vrtdataset.h" + +#include +#include +#include +#include + +/************************************************************************/ +/* VRTProcessedDatasetFunc */ +/************************************************************************/ + +//! Structure holding information for a VRTProcessedDataset function. +struct VRTProcessedDatasetFunc +{ + //! Processing function name + std::string osFuncName{}; + + //! User data to provide to pfnInit, pfnFree, pfnProcess callbacks. + void *pUserData = nullptr; + + //! Whether XML metadata has been specified + bool bMetadataSpecified = false; + + //! Map of (constant argument name, constant value) + std::map oMapConstantArguments{}; + + //! Set of builtin argument names (e.g "offset", "scale", "nodata") + std::set oSetBuiltinArguments{}; + + //! Arguments defined in the VRT + struct OtherArgument + { + std::string osType{}; + bool bRequired = false; + }; + + std::map oOtherArguments{}; + + //! Requested input data type. + GDALDataType eRequestedInputDT = GDT_Unknown; + + //! List of supported input datatypes. Empty if no restriction. + std::vector aeSupportedInputDT{}; + + //! List of supported input band counts. Empty if no restriction. + std::vector anSupportedInputBandCount{}; + + //! Optional initialization function + GDALVRTProcessedDatasetFuncInit pfnInit = nullptr; + + //! Optional free function + GDALVRTProcessedDatasetFuncFree pfnFree = nullptr; + + //! Required processing function + GDALVRTProcessedDatasetFuncProcess pfnProcess = nullptr; +}; + +/************************************************************************/ +/* GetGlobalMapProcessedDatasetFunc() */ +/************************************************************************/ + +/** Return the registry of VRTProcessedDatasetFunc functions */ +static std::map & +GetGlobalMapProcessedDatasetFunc() +{ + static std::map goMap; + return goMap; +} + +/************************************************************************/ +/* Step::~Step() */ +/************************************************************************/ + +/*! @cond Doxygen_Suppress */ + +/** Step destructor */ +VRTProcessedDataset::Step::~Step() +{ + deinit(); +} + +/************************************************************************/ +/* Step::deinit() */ +/************************************************************************/ + +/** Free pWorkingData */ +void VRTProcessedDataset::Step::deinit() +{ + if (pWorkingData) + { + const auto &oMapFunctions = GetGlobalMapProcessedDatasetFunc(); + const auto oIterFunc = oMapFunctions.find(osAlgorithm); + if (oIterFunc != oMapFunctions.end()) + { + if (oIterFunc->second.pfnFree) + { + oIterFunc->second.pfnFree(osAlgorithm.c_str(), + oIterFunc->second.pUserData, + pWorkingData); + } + } + else + { + CPLAssert(false); + } + pWorkingData = nullptr; + } +} + +/************************************************************************/ +/* Step::Step(Step&& other) */ +/************************************************************************/ + +/** Move constructor */ +VRTProcessedDataset::Step::Step(Step &&other) + : osAlgorithm(std::move(other.osAlgorithm)), + aosArguments(std::move(other.aosArguments)), eInDT(other.eInDT), + eOutDT(other.eOutDT), nInBands(other.nInBands), + nOutBands(other.nOutBands), adfInNoData(other.adfInNoData), + adfOutNoData(other.adfOutNoData), pWorkingData(other.pWorkingData) +{ + other.pWorkingData = nullptr; +} + +/************************************************************************/ +/* Step operator=(Step&& other) */ +/************************************************************************/ + +/** Move assignment operator */ +VRTProcessedDataset::Step &VRTProcessedDataset::Step::operator=(Step &&other) +{ + if (&other != this) + { + deinit(); + osAlgorithm = std::move(other.osAlgorithm); + aosArguments = std::move(other.aosArguments); + eInDT = other.eInDT; + eOutDT = other.eOutDT; + nInBands = other.nInBands; + nOutBands = other.nOutBands; + adfInNoData = std::move(other.adfInNoData); + adfOutNoData = std::move(other.adfOutNoData); + std::swap(pWorkingData, other.pWorkingData); + } + return *this; +} + +/************************************************************************/ +/* VRTProcessedDataset() */ +/************************************************************************/ + +/** Constructor */ +VRTProcessedDataset::VRTProcessedDataset(int nXSize, int nYSize) + : VRTDataset(nXSize, nYSize) +{ +} + +/************************************************************************/ +/* ~VRTProcessedDataset() */ +/************************************************************************/ + +VRTProcessedDataset::~VRTProcessedDataset() + +{ + VRTProcessedDataset::FlushCache(true); + VRTProcessedDataset::CloseDependentDatasets(); +} + +/************************************************************************/ +/* XMLInit() */ +/************************************************************************/ + +/** Instantiate object from XML tree */ +CPLErr VRTProcessedDataset::XMLInit(const CPLXMLNode *psTree, + const char *pszVRTPathIn) + +{ + if (Init(psTree, pszVRTPathIn, nullptr, nullptr, -1) != CE_None) + return CE_Failure; + + const auto poSrcFirstBand = m_poSrcDS->GetRasterBand(1); + const int nOvrCount = poSrcFirstBand->GetOverviewCount(); + for (int i = 0; i < nOvrCount; ++i) + { + auto poOvrDS = std::make_unique(0, 0); + if (poOvrDS->Init(psTree, pszVRTPathIn, this, m_poSrcDS.get(), i) != + CE_None) + break; + m_apoOverviewDatasets.emplace_back(std::move(poOvrDS)); + } + + return CE_None; +} + +/** Instantiate object from XML tree */ +CPLErr VRTProcessedDataset::Init(const CPLXMLNode *psTree, + const char *pszVRTPathIn, + const VRTProcessedDataset *poParentDS, + GDALDataset *poParentSrcDS, int iOvrLevel) + +{ + const CPLXMLNode *psInput = CPLGetXMLNode(psTree, "Input"); + if (!psInput) + { + CPLError(CE_Failure, CPLE_AppDefined, "Input element missing"); + return CE_Failure; + } + + if (pszVRTPathIn) + m_osVRTPath = pszVRTPathIn; + + if (poParentSrcDS) + { + m_poSrcDS.reset( + GDALCreateOverviewDataset(poParentSrcDS, iOvrLevel, true)); + } + else if (const CPLXMLNode *psSourceFileNameNode = + CPLGetXMLNode(psInput, "SourceFilename")) + { + const bool bRelativeToVRT = CPL_TO_BOOL( + atoi(CPLGetXMLValue(psSourceFileNameNode, "relativetoVRT", "0"))); + const std::string osFilename = VRTDataset::BuildSourceFilename( + CPLGetXMLValue(psInput, "SourceFilename", ""), pszVRTPathIn, + bRelativeToVRT); + m_poSrcDS.reset(GDALDataset::Open( + osFilename.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, nullptr, + nullptr, nullptr)); + } + else if (const CPLXMLNode *psVRTDataset = + CPLGetXMLNode(psInput, "VRTDataset")) + { + CPLXMLNode sVRTDatasetTmp = *psVRTDataset; + sVRTDatasetTmp.psNext = nullptr; + char *pszXML = CPLSerializeXMLTree(&sVRTDatasetTmp); + m_poSrcDS.reset(VRTDataset::OpenXML(pszXML, pszVRTPathIn, GA_ReadOnly)); + CPLFree(pszXML); + } + else + { + CPLError( + CE_Failure, CPLE_AppDefined, + "Input element should have a SourceFilename or VRTDataset element"); + return CE_Failure; + } + + if (!m_poSrcDS) + return CE_Failure; + + if (nRasterXSize == 0 && nRasterYSize == 0) + { + nRasterXSize = m_poSrcDS->GetRasterXSize(); + nRasterYSize = m_poSrcDS->GetRasterYSize(); + } + else if (nRasterXSize != m_poSrcDS->GetRasterXSize() || + nRasterYSize != m_poSrcDS->GetRasterYSize()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Inconsistent declared VRT dimensions with input dataset"); + return CE_Failure; + } + + if (m_poSrcDS->GetRasterCount() == 0) + return CE_Failure; + + // Inherit SRS from source if not explicitly defined in VRT + if (!CPLGetXMLNode(psTree, "SRS")) + { + const OGRSpatialReference *poSRS = m_poSrcDS->GetSpatialRef(); + if (poSRS) + { + m_poSRS.reset(poSRS->Clone()); + } + } + + // Inherit GeoTransform from source if not explicitly defined in VRT + if (iOvrLevel < 0 && !CPLGetXMLNode(psTree, "GeoTransform")) + { + if (m_poSrcDS->GetGeoTransform(m_adfGeoTransform) == CE_None) + m_bGeoTransformSet = true; + } + + /* -------------------------------------------------------------------- */ + /* Initialize blocksize before calling sub-init so that the */ + /* band initializers can get it from the dataset object when */ + /* they are created. */ + /* -------------------------------------------------------------------- */ + + const auto poSrcFirstBand = m_poSrcDS->GetRasterBand(1); + poSrcFirstBand->GetBlockSize(&m_nBlockXSize, &m_nBlockYSize); + if (const char *pszBlockXSize = + CPLGetXMLValue(psTree, "BlockXSize", nullptr)) + m_nBlockXSize = atoi(pszBlockXSize); + if (const char *pszBlockYSize = + CPLGetXMLValue(psTree, "BlockYSize", nullptr)) + m_nBlockYSize = atoi(pszBlockYSize); + + // Initialize all the general VRT stuff. + if (VRTDataset::XMLInit(psTree, pszVRTPathIn) != CE_None) + { + return CE_Failure; + } + + // Use geotransform from parent for overviews + if (iOvrLevel >= 0 && poParentDS->m_bGeoTransformSet) + { + m_bGeoTransformSet = true; + m_adfGeoTransform[0] = poParentDS->m_adfGeoTransform[0]; + m_adfGeoTransform[1] = poParentDS->m_adfGeoTransform[1]; + m_adfGeoTransform[2] = poParentDS->m_adfGeoTransform[2]; + m_adfGeoTransform[3] = poParentDS->m_adfGeoTransform[3]; + m_adfGeoTransform[4] = poParentDS->m_adfGeoTransform[4]; + m_adfGeoTransform[5] = poParentDS->m_adfGeoTransform[5]; + + m_adfGeoTransform[1] *= + static_cast(poParentDS->GetRasterXSize()) / nRasterXSize; + m_adfGeoTransform[2] *= + static_cast(poParentDS->GetRasterYSize()) / nRasterYSize; + m_adfGeoTransform[4] *= + static_cast(poParentDS->GetRasterXSize()) / nRasterXSize; + m_adfGeoTransform[5] *= + static_cast(poParentDS->GetRasterYSize()) / nRasterYSize; + } + + // Create bands automatically from source dataset if not explicitly defined + // in VRT. + if (!CPLGetXMLNode(psTree, "VRTRasterBand")) + { + for (int i = 0; i < m_poSrcDS->GetRasterCount(); ++i) + { + const auto poSrcBand = m_poSrcDS->GetRasterBand(i + 1); + auto poBand = new VRTProcessedRasterBand( + this, i + 1, poSrcBand->GetRasterDataType()); + poBand->CopyCommonInfoFrom(poSrcBand); + SetBand(i + 1, poBand); + } + } + + const CPLXMLNode *psProcessingSteps = + CPLGetXMLNode(psTree, "ProcessingSteps"); + if (!psProcessingSteps) + { + CPLError(CE_Failure, CPLE_AppDefined, + "ProcessingSteps element missing"); + return CE_Failure; + } + + const auto eInDT = poSrcFirstBand->GetRasterDataType(); + for (int i = 1; i < m_poSrcDS->GetRasterCount(); ++i) + { + const auto eDT = m_poSrcDS->GetRasterBand(i + 1)->GetRasterDataType(); + if (eDT != eInDT) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Not all bands of the input dataset have the same data " + "type. The data type of the first band will be used as " + "the reference one."); + break; + } + } + + GDALDataType eCurrentDT = eInDT; + int nCurrentBandCount = m_poSrcDS->GetRasterCount(); + + std::vector adfNoData; + for (int i = 1; i <= nCurrentBandCount; ++i) + { + int bHasVal = FALSE; + const double dfVal = + m_poSrcDS->GetRasterBand(i)->GetNoDataValue(&bHasVal); + adfNoData.emplace_back( + bHasVal ? dfVal : std::numeric_limits::quiet_NaN()); + } + + int nStepCount = 0; + for (const CPLXMLNode *psStep = psProcessingSteps->psChild; psStep; + psStep = psStep->psNext) + { + if (psStep->eType == CXT_Element && + strcmp(psStep->pszValue, "Step") == 0) + { + ++nStepCount; + } + } + + int iStep = 0; + for (const CPLXMLNode *psStep = psProcessingSteps->psChild; psStep; + psStep = psStep->psNext) + { + if (psStep->eType == CXT_Element && + strcmp(psStep->pszValue, "Step") == 0) + { + ++iStep; + const bool bIsFinalStep = (iStep == nStepCount); + std::vector adfOutNoData; + if (bIsFinalStep) + { + // Initialize adfOutNoData with nodata value of *output* bands + // for final step + for (int i = 1; i <= nBands; ++i) + { + int bHasVal = FALSE; + const double dfVal = + GetRasterBand(i)->GetNoDataValue(&bHasVal); + adfOutNoData.emplace_back( + bHasVal ? dfVal + : std::numeric_limits::quiet_NaN()); + } + } + if (!ParseStep(psStep, bIsFinalStep, eCurrentDT, nCurrentBandCount, + adfNoData, adfOutNoData)) + return CE_Failure; + adfNoData = std::move(adfOutNoData); + } + } + + if (m_aoSteps.empty()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "At least one step should be defined"); + return CE_Failure; + } + + if (nCurrentBandCount != nBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Number of output bands of last step is not consistent with " + "number of VRTProcessedRasterBand's"); + return CE_Failure; + } + + if (nBands > 1) + SetMetadataItem("INTERLEAVE", "PIXEL", "IMAGE_STRUCTURE"); + + m_oXMLTree.reset(CPLCloneXMLTree(psTree)); + + return CE_None; +} + +/************************************************************************/ +/* ParseStep() */ +/************************************************************************/ + +/** Parse the current Step node and create a corresponding entry in m_aoSteps. + * + * @param psStep Step node + * @param bIsFinalStep Whether this is the final step. + * @param[in,out] eCurrentDT Input data type for this step. + * Updated to output data type at end of method. + * @param[in,out] nCurrentBandCount Input band count for this step. + * Updated to output band cout at end of + * method. + * @param adfInNoData Input nodata values + * @param[in,out] adfOutNoData Output nodata values, to be filled by this + * method. When bIsFinalStep, this is also an + * input parameter. + * @return true on success. + */ +bool VRTProcessedDataset::ParseStep(const CPLXMLNode *psStep, bool bIsFinalStep, + GDALDataType &eCurrentDT, + int &nCurrentBandCount, + std::vector &adfInNoData, + std::vector &adfOutNoData) +{ + const char *pszStepName = CPLGetXMLValue( + psStep, "name", CPLSPrintf("nr %d", 1 + int(m_aoSteps.size()))); + const char *pszAlgorithm = CPLGetXMLValue(psStep, "Algorithm", nullptr); + if (!pszAlgorithm) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Step '%s' lacks a Algorithm element", pszStepName); + return false; + } + + const auto &oMapFunctions = GetGlobalMapProcessedDatasetFunc(); + const auto oIterFunc = oMapFunctions.find(pszAlgorithm); + if (oIterFunc == oMapFunctions.end()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Step '%s' uses unregistered algorithm '%s'", pszStepName, + pszAlgorithm); + return false; + } + + const auto &oFunc = oIterFunc->second; + + if (!oFunc.aeSupportedInputDT.empty()) + { + if (std::find(oFunc.aeSupportedInputDT.begin(), + oFunc.aeSupportedInputDT.end(), + eCurrentDT) == oFunc.aeSupportedInputDT.end()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Step '%s' (using algorithm '%s') does not " + "support input data type = '%s'", + pszStepName, pszAlgorithm, + GDALGetDataTypeName(eCurrentDT)); + return false; + } + } + + if (!oFunc.anSupportedInputBandCount.empty()) + { + if (std::find(oFunc.anSupportedInputBandCount.begin(), + oFunc.anSupportedInputBandCount.end(), + nCurrentBandCount) == + oFunc.anSupportedInputBandCount.end()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Step '%s' (using algorithm '%s') does not " + "support input band count = %d", + pszStepName, pszAlgorithm, nCurrentBandCount); + return false; + } + } + + Step oStep; + oStep.osAlgorithm = pszAlgorithm; + oStep.eInDT = oFunc.eRequestedInputDT != GDT_Unknown + ? oFunc.eRequestedInputDT + : eCurrentDT; + oStep.nInBands = nCurrentBandCount; + + // Unless modified by pfnInit... + oStep.eOutDT = oStep.eInDT; + + oStep.adfInNoData = adfInNoData; + oStep.adfOutNoData = bIsFinalStep ? adfOutNoData : adfInNoData; + + // Deal with constant arguments + for (const auto &nameValuePair : oFunc.oMapConstantArguments) + { + oStep.aosArguments.AddNameValue(nameValuePair.first.c_str(), + nameValuePair.second.c_str()); + } + + // Deal with built-in arguments + if (oFunc.oSetBuiltinArguments.find("nodata") != + oFunc.oSetBuiltinArguments.end()) + { + int bHasVal = false; + const auto poSrcFirstBand = m_poSrcDS->GetRasterBand(1); + const double dfVal = poSrcFirstBand->GetNoDataValue(&bHasVal); + if (bHasVal) + { + oStep.aosArguments.AddNameValue("nodata", + CPLSPrintf("%.18g", dfVal)); + } + } + + if (oFunc.oSetBuiltinArguments.find("offset_{band}") != + oFunc.oSetBuiltinArguments.end()) + { + for (int i = 1; i <= m_poSrcDS->GetRasterCount(); ++i) + { + int bHasVal = false; + const double dfVal = GetRasterBand(i)->GetOffset(&bHasVal); + oStep.aosArguments.AddNameValue( + CPLSPrintf("offset_%d", i), + CPLSPrintf("%.18g", bHasVal ? dfVal : 0.0)); + } + } + + if (oFunc.oSetBuiltinArguments.find("scale_{band}") != + oFunc.oSetBuiltinArguments.end()) + { + for (int i = 1; i <= m_poSrcDS->GetRasterCount(); ++i) + { + int bHasVal = false; + const double dfVal = GetRasterBand(i)->GetScale(&bHasVal); + oStep.aosArguments.AddNameValue( + CPLSPrintf("scale_%d", i), + CPLSPrintf("%.18g", bHasVal ? dfVal : 1.0)); + } + } + + // Parse arguments specified in VRT + std::set oFoundArguments; + + for (const CPLXMLNode *psStepChild = psStep->psChild; psStepChild; + psStepChild = psStepChild->psNext) + { + if (psStepChild->eType == CXT_Element && + strcmp(psStepChild->pszValue, "Argument") == 0) + { + const char *pszParamName = + CPLGetXMLValue(psStepChild, "name", nullptr); + if (!pszParamName) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Step '%s' has a Argument without a name attribute", + pszStepName); + return false; + } + const char *pszValue = CPLGetXMLValue(psStepChild, nullptr, ""); + auto oOtherArgIter = + oFunc.oOtherArguments.find(CPLString(pszParamName).tolower()); + if (!oFunc.oOtherArguments.empty() && + oOtherArgIter == oFunc.oOtherArguments.end()) + { + // If we got a parameter name like 'coefficients_1', + // try to fetch the generic 'coefficients_{band}' + std::string osParamName(pszParamName); + const auto nPos = osParamName.rfind('_'); + if (nPos != std::string::npos) + { + osParamName.resize(nPos + 1); + osParamName += "{band}"; + oOtherArgIter = oFunc.oOtherArguments.find( + CPLString(osParamName).tolower()); + } + } + if (oOtherArgIter != oFunc.oOtherArguments.end()) + { + oFoundArguments.insert(oOtherArgIter->first); + + const std::string &osType = oOtherArgIter->second.osType; + if (osType == "boolean") + { + if (!EQUAL(pszValue, "true") && !EQUAL(pszValue, "false")) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Step '%s' has a Argument '%s' whose " + "value '%s' is not a boolean", + pszStepName, pszParamName, pszValue); + return false; + } + } + else if (osType == "integer") + { + if (CPLGetValueType(pszValue) != CPL_VALUE_INTEGER) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Step '%s' has a Argument '%s' whose " + "value '%s' is not a integer", + pszStepName, pszParamName, pszValue); + return false; + } + } + else if (osType == "double") + { + const auto eType = CPLGetValueType(pszValue); + if (eType != CPL_VALUE_INTEGER && eType != CPL_VALUE_REAL) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Step '%s' has a Argument '%s' whose " + "value '%s' is not a double", + pszStepName, pszParamName, pszValue); + return false; + } + } + else if (osType == "double_list") + { + const CPLStringList aosTokens( + CSLTokenizeString2(pszValue, ",", 0)); + for (int i = 0; i < aosTokens.size(); ++i) + { + const auto eType = CPLGetValueType(aosTokens[i]); + if (eType != CPL_VALUE_INTEGER && + eType != CPL_VALUE_REAL) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Step '%s' has a Argument '%s' " + "whose value '%s' is not a " + "comma-separated list of doubles", + pszStepName, pszParamName, pszValue); + return false; + } + } + } + else if (osType != "string") + { + CPLDebug("VRT", "Unhandled argument type '%s'", + osType.c_str()); + CPLAssert(0); + } + } + else if (oFunc.bMetadataSpecified && + oFunc.oSetBuiltinArguments.find( + CPLString(pszParamName).tolower()) == + oFunc.oSetBuiltinArguments.end() && + oFunc.oMapConstantArguments.find( + CPLString(pszParamName).tolower()) == + oFunc.oMapConstantArguments.end()) + { + CPLError(CE_Warning, CPLE_NotSupported, + "Step '%s' has a Argument '%s' which is not " + "supported", + pszStepName, pszParamName); + } + + oStep.aosArguments.AddNameValue(pszParamName, pszValue); + } + } + + // Check that required arguments have been specified + for (const auto &oIter : oFunc.oOtherArguments) + { + if (oIter.second.bRequired && + oFoundArguments.find(oIter.first) == oFoundArguments.end()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Step '%s' lacks required Argument '%s'", pszStepName, + oIter.first.c_str()); + return false; + } + } + + if (oFunc.pfnInit) + { + double *padfOutNoData = nullptr; + if (bIsFinalStep) + { + oStep.nOutBands = nBands; + padfOutNoData = + static_cast(CPLMalloc(nBands * sizeof(double))); + CPLAssert(adfOutNoData.size() == static_cast(nBands)); + memcpy(padfOutNoData, adfOutNoData.data(), nBands * sizeof(double)); + } + else + { + oStep.nOutBands = 0; + } + + if (oFunc.pfnInit(pszAlgorithm, oFunc.pUserData, + oStep.aosArguments.List(), oStep.nInBands, + oStep.eInDT, adfInNoData.data(), &(oStep.nOutBands), + &(oStep.eOutDT), &padfOutNoData, m_osVRTPath.c_str(), + &(oStep.pWorkingData)) != CE_None) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Step '%s' (using algorithm '%s') init() function " + "failed", + pszStepName, pszAlgorithm); + CPLFree(padfOutNoData); + return false; + } + + // Input nodata values may have been modified by pfnInit() + oStep.adfInNoData = adfInNoData; + + if (padfOutNoData) + { + adfOutNoData = + std::vector(padfOutNoData, padfOutNoData + nBands); + } + else + { + adfOutNoData = std::vector( + oStep.nOutBands, std::numeric_limits::quiet_NaN()); + } + CPLFree(padfOutNoData); + + oStep.adfOutNoData = adfOutNoData; + } + else + { + oStep.nOutBands = oStep.nInBands; + adfOutNoData = oStep.adfOutNoData; + } + + eCurrentDT = oStep.eOutDT; + nCurrentBandCount = oStep.nOutBands; + + m_aoSteps.emplace_back(std::move(oStep)); + + return true; +} + +/************************************************************************/ +/* SerializeToXML() */ +/************************************************************************/ + +CPLXMLNode *VRTProcessedDataset::SerializeToXML(const char *pszVRTPathIn) + +{ + CPLXMLNode *psTree = CPLCloneXMLTree(m_oXMLTree.get()); + if (psTree == nullptr) + return psTree; + + /* -------------------------------------------------------------------- */ + /* Remove VRTRasterBand nodes from the original tree and find the */ + /* last child. */ + /* -------------------------------------------------------------------- */ + CPLXMLNode *psLastChild = psTree->psChild; + CPLXMLNode *psPrevChild = nullptr; + while (psLastChild) + { + CPLXMLNode *psNextChild = psLastChild->psNext; + if (psLastChild->eType == CXT_Element && + strcmp(psLastChild->pszValue, "VRTRasterBand") == 0) + { + if (psPrevChild) + psPrevChild->psNext = psNextChild; + else + psTree->psChild = psNextChild; + psLastChild->psNext = nullptr; + CPLDestroyXMLNode(psLastChild); + psLastChild = psPrevChild ? psPrevChild : psTree->psChild; + } + else if (!psNextChild) + { + break; + } + else + { + psPrevChild = psLastChild; + psLastChild = psNextChild; + } + } + CPLAssert(psLastChild); // we have at least Input + + /* -------------------------------------------------------------------- */ + /* Serialize bands. */ + /* -------------------------------------------------------------------- */ + bool bHasWarnedAboutRAMUsage = false; + size_t nAccRAMUsage = 0; + for (int iBand = 0; iBand < nBands; iBand++) + { + CPLXMLNode *psBandTree = + static_cast(papoBands[iBand]) + ->SerializeToXML(pszVRTPathIn, bHasWarnedAboutRAMUsage, + nAccRAMUsage); + + if (psBandTree != nullptr) + { + psLastChild->psNext = psBandTree; + psLastChild = psBandTree; + } + } + + return psTree; +} + +/************************************************************************/ +/* SerializeToXML() */ +/************************************************************************/ + +CPLXMLNode * +VRTProcessedRasterBand::SerializeToXML(const char *pszVRTPathIn, + bool &bHasWarnedAboutRAMUsage, + size_t &nAccRAMUsage) + +{ + CPLXMLNode *psTree = VRTRasterBand::SerializeToXML( + pszVRTPathIn, bHasWarnedAboutRAMUsage, nAccRAMUsage); + + /* -------------------------------------------------------------------- */ + /* Set subclass. */ + /* -------------------------------------------------------------------- */ + CPLCreateXMLNode(CPLCreateXMLNode(psTree, CXT_Attribute, "subClass"), + CXT_Text, "VRTProcessedRasterBand"); + + return psTree; +} + +/************************************************************************/ +/* GetBlockSize() */ +/************************************************************************/ + +/** Return block size */ +void VRTProcessedDataset::GetBlockSize(int *pnBlockXSize, + int *pnBlockYSize) const + +{ + *pnBlockXSize = m_nBlockXSize; + *pnBlockYSize = m_nBlockYSize; +} + +/************************************************************************/ +/* ProcessRegion() */ +/************************************************************************/ + +/** Compute pixel values for the specified region. + * + * The output is stored in m_abyInput in a pixel-interleaved way. + */ +bool VRTProcessedDataset::ProcessRegion(int nXOff, int nYOff, int nBufXSize, + int nBufYSize) +{ + + CPLAssert(!m_aoSteps.empty()); + + const int nFirstBandCount = m_aoSteps.front().nInBands; + CPLAssert(nFirstBandCount == m_poSrcDS->GetRasterCount()); + const GDALDataType eFirstDT = m_aoSteps.front().eInDT; + const int nFirstDTSize = GDALGetDataTypeSizeBytes(eFirstDT); + auto &abyInput = m_abyInput; + auto &abyOutput = m_abyOutput; + try + { + abyInput.resize(static_cast(nBufXSize) * nBufYSize * + nFirstBandCount * nFirstDTSize); + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating working buffer"); + return false; + } + + if (m_poSrcDS->RasterIO(GF_Read, nXOff, nYOff, nBufXSize, nBufYSize, + abyInput.data(), nBufXSize, nBufYSize, eFirstDT, + nFirstBandCount, nullptr, + nFirstDTSize * nFirstBandCount, + nFirstDTSize * nFirstBandCount * nBufXSize, + nFirstDTSize, nullptr) != CE_None) + { + return false; + } + + const double dfSrcXOff = nXOff; + const double dfSrcYOff = nYOff; + const double dfSrcXSize = nBufXSize; + const double dfSrcYSize = nBufYSize; + + double adfSrcGT[6]; + if (m_poSrcDS->GetGeoTransform(adfSrcGT) != CE_None) + { + adfSrcGT[0] = 0; + adfSrcGT[1] = 1; + adfSrcGT[2] = 0; + adfSrcGT[3] = 0; + adfSrcGT[4] = 0; + adfSrcGT[5] = 1; + } + + GDALDataType eLastDT = eFirstDT; + const auto &oMapFunctions = GetGlobalMapProcessedDatasetFunc(); + for (const auto &oStep : m_aoSteps) + { + const auto oIterFunc = oMapFunctions.find(oStep.osAlgorithm); + CPLAssert(oIterFunc != oMapFunctions.end()); + + // Data type adaptation + if (eLastDT != oStep.eInDT) + { + try + { + abyOutput.resize(static_cast(nBufXSize) * nBufYSize * + oStep.nInBands * + GDALGetDataTypeSizeBytes(oStep.eInDT)); + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating working buffer"); + return false; + } + + GDALCopyWords64(abyInput.data(), eLastDT, + GDALGetDataTypeSizeBytes(eLastDT), abyOutput.data(), + oStep.eInDT, GDALGetDataTypeSizeBytes(oStep.eInDT), + static_cast(nBufXSize) * nBufYSize * + oStep.nInBands); + + std::swap(abyInput, abyOutput); + } + + try + { + abyOutput.resize(static_cast(nBufXSize) * nBufYSize * + oStep.nOutBands * + GDALGetDataTypeSizeBytes(oStep.eOutDT)); + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating working buffer"); + return false; + } + + const auto &oFunc = oIterFunc->second; + if (oFunc.pfnProcess( + oStep.osAlgorithm.c_str(), oFunc.pUserData, oStep.pWorkingData, + oStep.aosArguments.List(), nBufXSize, nBufYSize, + abyInput.data(), abyInput.size(), oStep.eInDT, oStep.nInBands, + oStep.adfInNoData.data(), abyOutput.data(), abyOutput.size(), + oStep.eOutDT, oStep.nOutBands, oStep.adfOutNoData.data(), + dfSrcXOff, dfSrcYOff, dfSrcXSize, dfSrcYSize, adfSrcGT, + m_osVRTPath.c_str(), + /*papszExtra=*/nullptr) != CE_None) + { + return false; + } + + std::swap(abyInput, abyOutput); + eLastDT = oStep.eOutDT; + } + + return true; +} + +/************************************************************************/ +/* VRTProcessedRasterBand() */ +/************************************************************************/ + +/** Constructor */ +VRTProcessedRasterBand::VRTProcessedRasterBand(VRTProcessedDataset *poDSIn, + int nBandIn, + GDALDataType eDataTypeIn) +{ + Initialize(poDSIn->GetRasterXSize(), poDSIn->GetRasterYSize()); + + poDS = poDSIn; + nBand = nBandIn; + eAccess = GA_Update; + eDataType = eDataTypeIn; + + poDSIn->GetBlockSize(&nBlockXSize, &nBlockYSize); +} + +/************************************************************************/ +/* GetOverviewCount() */ +/************************************************************************/ + +int VRTProcessedRasterBand::GetOverviewCount() +{ + auto poVRTDS = cpl::down_cast(poDS); + return static_cast(poVRTDS->m_apoOverviewDatasets.size()); +} + +/************************************************************************/ +/* GetOverview() */ +/************************************************************************/ + +GDALRasterBand *VRTProcessedRasterBand::GetOverview(int iOvr) +{ + auto poVRTDS = cpl::down_cast(poDS); + if (iOvr < 0 || + iOvr >= static_cast(poVRTDS->m_apoOverviewDatasets.size())) + return nullptr; + return poVRTDS->m_apoOverviewDatasets[iOvr]->GetRasterBand(nBand); +} + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr VRTProcessedRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, + void *pImage) + +{ + auto poVRTDS = cpl::down_cast(poDS); + + int nBufXSize = 0; + int nBufYSize = 0; + GetActualBlockSize(nBlockXOff, nBlockYOff, &nBufXSize, &nBufYSize); + + const int nXPixelOff = nBlockXOff * nBlockXSize; + const int nYPixelOff = nBlockYOff * nBlockYSize; + if (!poVRTDS->ProcessRegion(nXPixelOff, nYPixelOff, nBufXSize, nBufYSize)) + { + return CE_Failure; + } + + const int nOutBands = poVRTDS->m_aoSteps.back().nOutBands; + CPLAssert(nOutBands == poVRTDS->GetRasterCount()); + const auto eLastDT = poVRTDS->m_aoSteps.back().eOutDT; + const int nLastDTSize = GDALGetDataTypeSizeBytes(eLastDT); + const int nDTSize = GDALGetDataTypeSizeBytes(eDataType); + + // Dispatch final output buffer to cached blocks of output bands + for (int iDstBand = 0; iDstBand < nOutBands; ++iDstBand) + { + GDALRasterBlock *poBlock = nullptr; + GByte *pDst; + if (iDstBand + 1 == nBand) + { + pDst = static_cast(pImage); + } + else + { + auto poOtherBand = poVRTDS->papoBands[iDstBand]; + poBlock = poOtherBand->TryGetLockedBlockRef(nBlockXOff, nBlockYOff); + if (poBlock) + { + poBlock->DropLock(); + continue; + } + poBlock = poOtherBand->GetLockedBlockRef( + nBlockXOff, nBlockYOff, /* bJustInitialized = */ true); + if (!poBlock) + continue; + pDst = static_cast(poBlock->GetDataRef()); + } + for (int iY = 0; iY < nBufYSize; ++iY) + { + GDALCopyWords(poVRTDS->m_abyInput.data() + + (iDstBand + static_cast(iY) * nBufXSize * + nOutBands) * + nLastDTSize, + eLastDT, nLastDTSize * nOutBands, + pDst + + static_cast(iY) * nBlockXSize * nDTSize, + eDataType, nDTSize, nBufXSize); + } + if (poBlock) + poBlock->DropLock(); + } + + return CE_None; +} + +/*! @endcond */ + +/************************************************************************/ +/* GDALVRTRegisterProcessedDatasetFunc() */ +/************************************************************************/ + +/** Register a function to be used by VRTProcessedDataset. + + An example of content for pszXMLMetadata is: + \verbatim + + + + + + + + \endverbatim + + @param pszFuncName Function name. Must be unique and not null. + @param pUserData User data. May be nullptr. Must remain valid during the + lifetime of GDAL. + @param pszXMLMetadata XML metadata describing the function arguments. May be + nullptr if there are no arguments. + @param eRequestedInputDT If the pfnProcess callback only supports a single + data type, it should be specified in this parameter. + Otherwise set it to GDT_Unknown. + @param paeSupportedInputDT List of supported input data types. May be nullptr + if all are supported or if eRequestedInputDT is + set to a non GDT_Unknown value. + @param nSupportedInputDTSize Size of paeSupportedInputDT + @param panSupportedInputBandCount List of supported band count. May be nullptr + if any source band count is supported. + @param nSupportedInputBandCountSize Size of panSupportedInputBandCount + @param pfnInit Initialization function called when a VRTProcessedDataset + step uses the register function. This initialization function + will return the output data type, output band count and + potentially initialize a working structure, typically parsing + arguments. May be nullptr. + If not specified, it will be assumed that the input and output + data types are the same, and that the input number of bands + and output number of bands are the same. + @param pfnFree Free function that will free the working structure allocated + by pfnInit. May be nullptr. + @param pfnProcess Processing function called to compute pixel values. Must + not be nullptr. + @param papszOptions Unused currently. Must be nullptr. + @return CE_None in case of success, error otherwise. + @since 3.9 + */ +CPLErr GDALVRTRegisterProcessedDatasetFunc( + const char *pszFuncName, void *pUserData, const char *pszXMLMetadata, + GDALDataType eRequestedInputDT, const GDALDataType *paeSupportedInputDT, + size_t nSupportedInputDTSize, const int *panSupportedInputBandCount, + size_t nSupportedInputBandCountSize, + GDALVRTProcessedDatasetFuncInit pfnInit, + GDALVRTProcessedDatasetFuncFree pfnFree, + GDALVRTProcessedDatasetFuncProcess pfnProcess, + CPL_UNUSED CSLConstList papszOptions) +{ + if (pszFuncName == nullptr || pszFuncName[0] == '\0') + { + CPLError(CE_Failure, CPLE_AppDefined, + "pszFuncName should be non-empty"); + return CE_Failure; + } + + auto &oMap = GetGlobalMapProcessedDatasetFunc(); + if (oMap.find(pszFuncName) != oMap.end()) + { + CPLError(CE_Failure, CPLE_AppDefined, "%s already registered", + pszFuncName); + return CE_Failure; + } + + if (!pfnProcess) + { + CPLError(CE_Failure, CPLE_AppDefined, "pfnProcess should not be null"); + return CE_Failure; + } + + VRTProcessedDatasetFunc oFunc; + oFunc.osFuncName = pszFuncName; + oFunc.pUserData = pUserData; + if (pszXMLMetadata) + { + oFunc.bMetadataSpecified = true; + auto psTree = CPLXMLTreeCloser(CPLParseXMLString(pszXMLMetadata)); + if (!psTree) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot parse pszXMLMetadata=%s for %s", pszXMLMetadata, + pszFuncName); + return CE_Failure; + } + const CPLXMLNode *psRoot = CPLGetXMLNode( + psTree.get(), "=ProcessedDatasetFunctionArgumentsList"); + if (!psRoot) + { + CPLError(CE_Failure, CPLE_AppDefined, + "No root ProcessedDatasetFunctionArgumentsList element in " + "pszXMLMetadata=%s for %s", + pszXMLMetadata, pszFuncName); + return CE_Failure; + } + for (const CPLXMLNode *psIter = psRoot->psChild; psIter; + psIter = psIter->psNext) + { + if (psIter->eType == CXT_Element && + strcmp(psIter->pszValue, "Argument") == 0) + { + const char *pszName = CPLGetXMLValue(psIter, "name", nullptr); + if (!pszName) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing Argument.name attribute in " + "pszXMLMetadata=%s for %s", + pszXMLMetadata, pszFuncName); + return CE_Failure; + } + const char *pszType = CPLGetXMLValue(psIter, "type", nullptr); + if (!pszType) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing Argument.type attribute in " + "pszXMLMetadata=%s for %s", + pszXMLMetadata, pszFuncName); + return CE_Failure; + } + if (strcmp(pszType, "constant") == 0) + { + const char *pszValue = + CPLGetXMLValue(psIter, "value", nullptr); + if (!pszValue) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing Argument.value attribute in " + "pszXMLMetadata=%s for %s", + pszXMLMetadata, pszFuncName); + return CE_Failure; + } + oFunc.oMapConstantArguments[CPLString(pszName).tolower()] = + pszValue; + } + else if (strcmp(pszType, "builtin") == 0) + { + if (EQUAL(pszName, "nodata") || + EQUAL(pszName, "offset_{band}") || + EQUAL(pszName, "scale_{band}")) + { + oFunc.oSetBuiltinArguments.insert( + CPLString(pszName).tolower()); + } + else + { + CPLError(CE_Failure, CPLE_NotSupported, + "Unsupported builtin parameter name %s in " + "pszXMLMetadata=%s for %s. Only nodata, " + "offset_{band} and scale_{band} are supported", + pszName, pszXMLMetadata, pszFuncName); + return CE_Failure; + } + } + else if (strcmp(pszType, "boolean") == 0 || + strcmp(pszType, "string") == 0 || + strcmp(pszType, "integer") == 0 || + strcmp(pszType, "double") == 0 || + strcmp(pszType, "double_list") == 0) + { + VRTProcessedDatasetFunc::OtherArgument otherArgument; + otherArgument.bRequired = CPLTestBool( + CPLGetXMLValue(psIter, "required", "false")); + otherArgument.osType = pszType; + oFunc.oOtherArguments[CPLString(pszName).tolower()] = + std::move(otherArgument); + } + else + { + CPLError(CE_Failure, CPLE_NotSupported, + "Unsupported type for parameter %s in " + "pszXMLMetadata=%s for %s. Only boolean, string, " + "integer, double and double_list are supported", + pszName, pszXMLMetadata, pszFuncName); + return CE_Failure; + } + } + } + } + oFunc.eRequestedInputDT = eRequestedInputDT; + if (nSupportedInputDTSize) + { + oFunc.aeSupportedInputDT.insert( + oFunc.aeSupportedInputDT.end(), paeSupportedInputDT, + paeSupportedInputDT + nSupportedInputDTSize); + } + if (nSupportedInputBandCountSize) + { + oFunc.anSupportedInputBandCount.insert( + oFunc.anSupportedInputBandCount.end(), panSupportedInputBandCount, + panSupportedInputBandCount + nSupportedInputBandCountSize); + } + oFunc.pfnInit = pfnInit; + oFunc.pfnFree = pfnFree; + oFunc.pfnProcess = pfnProcess; + + oMap[pszFuncName] = std::move(oFunc); + + return CE_None; +} diff --git a/frmts/vrt/vrtprocesseddatasetfunctions.cpp b/frmts/vrt/vrtprocesseddatasetfunctions.cpp new file mode 100644 index 000000000000..1513641b9445 --- /dev/null +++ b/frmts/vrt/vrtprocesseddatasetfunctions.cpp @@ -0,0 +1,1579 @@ +/****************************************************************************** + * + * Project: Virtual GDAL Datasets + * Purpose: Implementation of VRTProcessedDataset processing functions + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "cpl_minixml.h" +#include "cpl_string.h" +#include "vrtdataset.h" + +#include +#include +#include +#include +#include + +/************************************************************************/ +/* GetDstValue() */ +/************************************************************************/ + +/** Return a destination value given an initial value, the destination no data + * value and its replacement value + */ +static inline double GetDstValue(double dfVal, double dfDstNoData, + double dfReplacementDstNodata, + GDALDataType eIntendedDstDT, + bool bDstIntendedDTIsInteger) +{ + if (bDstIntendedDTIsInteger && std::round(dfVal) == dfDstNoData) + { + return dfReplacementDstNodata; + } + else if (eIntendedDstDT == GDT_Float32 && + static_cast(dfVal) == static_cast(dfDstNoData)) + { + return dfReplacementDstNodata; + } + else if (eIntendedDstDT == GDT_Float64 && dfVal == dfDstNoData) + { + return dfReplacementDstNodata; + } + else + { + return dfVal; + } +} + +/************************************************************************/ +/* BandAffineCombinationData */ +/************************************************************************/ + +namespace +{ +/** Working structure for 'BandAffineCombination' builtin function. */ +struct BandAffineCombinationData +{ + static constexpr const char *const EXPECTED_SIGNATURE = + "BandAffineCombination"; + //! Signature (to make sure callback functions are called with the right argument) + const std::string m_osSignature = EXPECTED_SIGNATURE; + + /** Replacement nodata value */ + std::vector m_adfReplacementDstNodata{}; + + /** Intended destination data type. */ + GDALDataType m_eIntendedDstDT = GDT_Float64; + + /** Affine transformation coefficients. + * m_aadfCoefficients[i][0] is the constant term for the i(th) dst band + * m_aadfCoefficients[i][j] is the weight of the j(th) src band for the + * i(th) dst vand. + * Said otherwise dst[i] = m_aadfCoefficients[i][0] + + * sum(m_aadfCoefficients[i][j + 1] * src[j] for j in 0...nSrcBands-1) + */ + std::vector> m_aadfCoefficients{}; + + //! Minimum clamping value. + double m_dfClampMin = std::numeric_limits::quiet_NaN(); + + //! Maximum clamping value. + double m_dfClampMax = std::numeric_limits::quiet_NaN(); +}; +} // namespace + +/************************************************************************/ +/* SetOutputValuesForInNoDataAndOutNoData() */ +/************************************************************************/ + +static std::vector SetOutputValuesForInNoDataAndOutNoData( + int nInBands, double *padfInNoData, int *pnOutBands, + double **ppadfOutNoData, bool bSrcNodataSpecified, double dfSrcNoData, + bool bDstNodataSpecified, double dfDstNoData, bool bIsFinalStep) +{ + if (bSrcNodataSpecified) + { + std::vector adfNoData(nInBands, dfSrcNoData); + memcpy(padfInNoData, adfNoData.data(), + adfNoData.size() * sizeof(double)); + } + + std::vector adfDstNoData; + if (bDstNodataSpecified) + { + adfDstNoData.resize(*pnOutBands, dfDstNoData); + } + else if (bIsFinalStep) + { + adfDstNoData = + std::vector(*ppadfOutNoData, *ppadfOutNoData + *pnOutBands); + } + else + { + adfDstNoData = + std::vector(padfInNoData, padfInNoData + nInBands); + adfDstNoData.resize(*pnOutBands, *padfInNoData); + } + + if (*ppadfOutNoData == nullptr) + { + *ppadfOutNoData = + static_cast(CPLMalloc(*pnOutBands * sizeof(double))); + } + memcpy(*ppadfOutNoData, adfDstNoData.data(), *pnOutBands * sizeof(double)); + + return adfDstNoData; +} + +/************************************************************************/ +/* BandAffineCombinationInit() */ +/************************************************************************/ + +/** Init function for 'BandAffineCombination' builtin function. */ +static CPLErr BandAffineCombinationInit( + const char * /*pszFuncName*/, void * /*pUserData*/, + CSLConstList papszFunctionArgs, int nInBands, GDALDataType eInDT, + double *padfInNoData, int *pnOutBands, GDALDataType *peOutDT, + double **ppadfOutNoData, const char * /* pszVRTPath */, + VRTPDWorkingDataPtr *ppWorkingData) +{ + CPLAssert(eInDT == GDT_Float64); + + *peOutDT = eInDT; + *ppWorkingData = nullptr; + + auto data = std::make_unique(); + + std::map> oMapCoefficients{}; + double dfSrcNoData = std::numeric_limits::quiet_NaN(); + bool bSrcNodataSpecified = false; + double dfDstNoData = std::numeric_limits::quiet_NaN(); + bool bDstNodataSpecified = false; + double dfReplacementDstNodata = std::numeric_limits::quiet_NaN(); + bool bReplacementDstNodataSpecified = false; + + for (const auto &[pszKey, pszValue] : + cpl::IterateNameValue(papszFunctionArgs)) + { + if (EQUAL(pszKey, "src_nodata")) + { + bSrcNodataSpecified = true; + dfSrcNoData = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "dst_nodata")) + { + bDstNodataSpecified = true; + dfDstNoData = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "replacement_nodata")) + { + bReplacementDstNodataSpecified = true; + dfReplacementDstNodata = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "dst_intended_datatype")) + { + for (GDALDataType eDT = GDT_Byte; eDT < GDT_TypeCount; + eDT = static_cast(eDT + 1)) + { + if (EQUAL(GDALGetDataTypeName(eDT), pszValue)) + { + data->m_eIntendedDstDT = eDT; + break; + } + } + } + else if (STARTS_WITH_CI(pszKey, "coefficients_")) + { + const int nTargetBand = atoi(pszKey + strlen("coefficients_")); + if (nTargetBand <= 0 || nTargetBand > 65536) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + const CPLStringList aosTokens(CSLTokenizeString2(pszValue, ",", 0)); + if (aosTokens.size() != 1 + nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Argument %s has %d values, whereas %d are expected", + pszKey, aosTokens.size(), 1 + nInBands); + return CE_Failure; + } + std::vector adfValues; + for (int i = 0; i < aosTokens.size(); ++i) + { + adfValues.push_back(CPLAtof(aosTokens[i])); + } + oMapCoefficients[nTargetBand - 1] = std::move(adfValues); + } + else if (EQUAL(pszKey, "min")) + { + data->m_dfClampMin = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "max")) + { + data->m_dfClampMax = CPLAtof(pszValue); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Unrecognized argument name %s. Ignored", pszKey); + } + } + + const bool bIsFinalStep = *pnOutBands != 0; + if (bIsFinalStep) + { + if (*pnOutBands != static_cast(oMapCoefficients.size())) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Final step expect %d bands, but only %d coefficient_XX " + "are provided", + *pnOutBands, static_cast(oMapCoefficients.size())); + return CE_Failure; + } + } + else + { + *pnOutBands = static_cast(oMapCoefficients.size()); + } + + const std::vector adfDstNoData = + SetOutputValuesForInNoDataAndOutNoData( + nInBands, padfInNoData, pnOutBands, ppadfOutNoData, + bSrcNodataSpecified, dfSrcNoData, bDstNodataSpecified, dfDstNoData, + bIsFinalStep); + + if (bReplacementDstNodataSpecified) + { + data->m_adfReplacementDstNodata.resize(*pnOutBands, + dfReplacementDstNodata); + } + else + { + for (double dfVal : adfDstNoData) + { + data->m_adfReplacementDstNodata.emplace_back( + GDALGetNoDataReplacementValue(data->m_eIntendedDstDT, dfVal)); + } + } + + // Check we have a set of coefficient for all output bands and + // convert the map to a vector + for (auto &oIter : oMapCoefficients) + { + const int iExpected = static_cast(data->m_aadfCoefficients.size()); + if (oIter.first != iExpected) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Argument coefficients_%d is missing", iExpected + 1); + return CE_Failure; + } + data->m_aadfCoefficients.emplace_back(std::move(oIter.second)); + } + *ppWorkingData = data.release(); + return CE_None; +} + +/************************************************************************/ +/* BandAffineCombinationFree() */ +/************************************************************************/ + +/** Free function for 'BandAffineCombination' builtin function. */ +static void BandAffineCombinationFree(const char * /*pszFuncName*/, + void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData) +{ + BandAffineCombinationData *data = + static_cast(pWorkingData); + CPLAssert(data->m_osSignature == + BandAffineCombinationData::EXPECTED_SIGNATURE); + CPL_IGNORE_RET_VAL(data->m_osSignature); + delete data; +} + +/************************************************************************/ +/* BandAffineCombinationProcess() */ +/************************************************************************/ + +/** Processing function for 'BandAffineCombination' builtin function. */ +static CPLErr BandAffineCombinationProcess( + const char * /*pszFuncName*/, void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData, CSLConstList /* papszFunctionArgs*/, + int nBufXSize, int nBufYSize, const void *pInBuffer, size_t nInBufferSize, + GDALDataType eInDT, int nInBands, const double *CPL_RESTRICT padfInNoData, + void *pOutBuffer, size_t nOutBufferSize, GDALDataType eOutDT, int nOutBands, + const double *CPL_RESTRICT padfOutNoData, double /*dfSrcXOff*/, + double /*dfSrcYOff*/, double /*dfSrcXSize*/, double /*dfSrcYSize*/, + const double /*adfSrcGT*/[], const char * /* pszVRTPath */, + CSLConstList /*papszExtra*/) +{ + const size_t nElts = static_cast(nBufXSize) * nBufYSize; + + CPL_IGNORE_RET_VAL(eInDT); + CPLAssert(eInDT == GDT_Float64); + CPL_IGNORE_RET_VAL(eOutDT); + CPLAssert(eOutDT == GDT_Float64); + CPL_IGNORE_RET_VAL(nInBufferSize); + CPLAssert(nInBufferSize == nElts * nInBands * sizeof(double)); + CPL_IGNORE_RET_VAL(nOutBufferSize); + CPLAssert(nOutBufferSize == nElts * nOutBands * sizeof(double)); + + const BandAffineCombinationData *data = + static_cast(pWorkingData); + CPLAssert(data->m_osSignature == + BandAffineCombinationData::EXPECTED_SIGNATURE); + const double *CPL_RESTRICT padfSrc = static_cast(pInBuffer); + double *CPL_RESTRICT padfDst = static_cast(pOutBuffer); + const bool bDstIntendedDTIsInteger = + GDALDataTypeIsInteger(data->m_eIntendedDstDT); + const double dfClampMin = data->m_dfClampMin; + const double dfClampMax = data->m_dfClampMax; + for (size_t i = 0; i < nElts; ++i) + { + for (int iDst = 0; iDst < nOutBands; ++iDst) + { + const auto &adfCoefficients = data->m_aadfCoefficients[iDst]; + double dfVal = adfCoefficients[0]; + bool bSetNoData = false; + for (int iSrc = 0; iSrc < nInBands; ++iSrc) + { + // written this way to work with a NaN value + if (!(padfSrc[iSrc] != padfInNoData[iSrc])) + { + bSetNoData = true; + break; + } + dfVal += adfCoefficients[iSrc + 1] * padfSrc[iSrc]; + } + if (bSetNoData) + { + *padfDst = padfOutNoData[iDst]; + } + else + { + double dfDstVal = GetDstValue( + dfVal, padfOutNoData[iDst], + data->m_adfReplacementDstNodata[iDst], + data->m_eIntendedDstDT, bDstIntendedDTIsInteger); + if (dfDstVal < dfClampMin) + dfDstVal = dfClampMin; + if (dfDstVal > dfClampMax) + dfDstVal = dfClampMax; + *padfDst = dfDstVal; + } + ++padfDst; + } + padfSrc += nInBands; + } + + return CE_None; +} + +/************************************************************************/ +/* LUTData */ +/************************************************************************/ + +namespace +{ +/** Working structure for 'LUT' builtin function. */ +struct LUTData +{ + static constexpr const char *const EXPECTED_SIGNATURE = "LUT"; + //! Signature (to make sure callback functions are called with the right argument) + const std::string m_osSignature = EXPECTED_SIGNATURE; + + //! m_aadfLUTInputs[i][j] is the j(th) input value for that LUT of band i. + std::vector> m_aadfLUTInputs{}; + + //! m_aadfLUTOutputs[i][j] is the j(th) output value for that LUT of band i. + std::vector> m_aadfLUTOutputs{}; + + /************************************************************************/ + /* LookupValue() */ + /************************************************************************/ + + double LookupValue(int iBand, double dfInput) const + { + const auto &adfInput = m_aadfLUTInputs[iBand]; + const auto &afdOutput = m_aadfLUTOutputs[iBand]; + + // Find the index of the first element in the LUT input array that + // is not smaller than the input value. + int i = static_cast( + std::lower_bound(adfInput.data(), adfInput.data() + adfInput.size(), + dfInput) - + adfInput.data()); + + if (i == 0) + return afdOutput[0]; + + // If the index is beyond the end of the LUT input array, the input + // value is larger than all the values in the array. + if (i == static_cast(adfInput.size())) + return afdOutput.back(); + + if (adfInput[i] == dfInput) + return afdOutput[i]; + + // Otherwise, interpolate. + return afdOutput[i - 1] + (dfInput - adfInput[i - 1]) * + ((afdOutput[i] - afdOutput[i - 1]) / + (adfInput[i] - adfInput[i - 1])); + } +}; +} // namespace + +/************************************************************************/ +/* LUTInit() */ +/************************************************************************/ + +/** Init function for 'LUT' builtin function. */ +static CPLErr LUTInit(const char * /*pszFuncName*/, void * /*pUserData*/, + CSLConstList papszFunctionArgs, int nInBands, + GDALDataType eInDT, double *padfInNoData, int *pnOutBands, + GDALDataType *peOutDT, double **ppadfOutNoData, + const char * /* pszVRTPath */, + VRTPDWorkingDataPtr *ppWorkingData) +{ + CPLAssert(eInDT == GDT_Float64); + + const bool bIsFinalStep = *pnOutBands != 0; + *peOutDT = eInDT; + *ppWorkingData = nullptr; + + if (!bIsFinalStep) + { + *pnOutBands = nInBands; + } + + auto data = std::make_unique(); + + double dfSrcNoData = std::numeric_limits::quiet_NaN(); + bool bSrcNodataSpecified = false; + double dfDstNoData = std::numeric_limits::quiet_NaN(); + bool bDstNodataSpecified = false; + + std::map, std::vector>> oMap{}; + + for (const auto &[pszKey, pszValue] : + cpl::IterateNameValue(papszFunctionArgs)) + { + if (EQUAL(pszKey, "src_nodata")) + { + bSrcNodataSpecified = true; + dfSrcNoData = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "dst_nodata")) + { + bDstNodataSpecified = true; + dfDstNoData = CPLAtof(pszValue); + } + else if (STARTS_WITH_CI(pszKey, "lut_")) + { + const int nBand = atoi(pszKey + strlen("lut_")); + if (nBand <= 0 || nBand > nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + const CPLStringList aosTokens(CSLTokenizeString2(pszValue, ",", 0)); + std::vector adfInputValues; + std::vector adfOutputValues; + for (int i = 0; i < aosTokens.size(); ++i) + { + const CPLStringList aosTokens2( + CSLTokenizeString2(aosTokens[i], ":", 0)); + if (aosTokens2.size() != 2) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid value for argument '%s'", pszKey); + return CE_Failure; + } + adfInputValues.push_back(CPLAtof(aosTokens2[0])); + adfOutputValues.push_back(CPLAtof(aosTokens2[1])); + } + oMap[nBand - 1] = std::pair(std::move(adfInputValues), + std::move(adfOutputValues)); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Unrecognized argument name %s. Ignored", pszKey); + } + } + + SetOutputValuesForInNoDataAndOutNoData( + nInBands, padfInNoData, pnOutBands, ppadfOutNoData, bSrcNodataSpecified, + dfSrcNoData, bDstNodataSpecified, dfDstNoData, bIsFinalStep); + + int iExpected = 0; + // Check we have values for all bands and convert to vector + for (auto &oIter : oMap) + { + if (oIter.first != iExpected) + { + CPLError(CE_Failure, CPLE_AppDefined, "Argument lut_%d is missing", + iExpected + 1); + return CE_Failure; + } + ++iExpected; + data->m_aadfLUTInputs.emplace_back(std::move(oIter.second.first)); + data->m_aadfLUTOutputs.emplace_back(std::move(oIter.second.second)); + } + + if (static_cast(oMap.size()) < *pnOutBands) + { + CPLError(CE_Failure, CPLE_AppDefined, "Missing lut_XX element(s)"); + return CE_Failure; + } + + *ppWorkingData = data.release(); + return CE_None; +} + +/************************************************************************/ +/* LUTFree() */ +/************************************************************************/ + +/** Free function for 'LUT' builtin function. */ +static void LUTFree(const char * /*pszFuncName*/, void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData) +{ + LUTData *data = static_cast(pWorkingData); + CPLAssert(data->m_osSignature == LUTData::EXPECTED_SIGNATURE); + CPL_IGNORE_RET_VAL(data->m_osSignature); + delete data; +} + +/************************************************************************/ +/* LUTProcess() */ +/************************************************************************/ + +/** Processing function for 'LUT' builtin function. */ +static CPLErr +LUTProcess(const char * /*pszFuncName*/, void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData, + CSLConstList /* papszFunctionArgs*/, int nBufXSize, int nBufYSize, + const void *pInBuffer, size_t nInBufferSize, GDALDataType eInDT, + int nInBands, const double *CPL_RESTRICT padfInNoData, + void *pOutBuffer, size_t nOutBufferSize, GDALDataType eOutDT, + int nOutBands, const double *CPL_RESTRICT padfOutNoData, + double /*dfSrcXOff*/, double /*dfSrcYOff*/, double /*dfSrcXSize*/, + double /*dfSrcYSize*/, const double /*adfSrcGT*/[], + const char * /* pszVRTPath */, CSLConstList /*papszExtra*/) +{ + const size_t nElts = static_cast(nBufXSize) * nBufYSize; + + CPL_IGNORE_RET_VAL(eInDT); + CPLAssert(eInDT == GDT_Float64); + CPL_IGNORE_RET_VAL(eOutDT); + CPLAssert(eOutDT == GDT_Float64); + CPL_IGNORE_RET_VAL(nInBufferSize); + CPLAssert(nInBufferSize == nElts * nInBands * sizeof(double)); + CPL_IGNORE_RET_VAL(nOutBufferSize); + CPLAssert(nOutBufferSize == nElts * nOutBands * sizeof(double)); + CPLAssert(nInBands == nOutBands); + CPL_IGNORE_RET_VAL(nOutBands); + + const LUTData *data = static_cast(pWorkingData); + CPLAssert(data->m_osSignature == LUTData::EXPECTED_SIGNATURE); + const double *CPL_RESTRICT padfSrc = static_cast(pInBuffer); + double *CPL_RESTRICT padfDst = static_cast(pOutBuffer); + for (size_t i = 0; i < nElts; ++i) + { + for (int iBand = 0; iBand < nInBands; ++iBand) + { + // written this way to work with a NaN value + if (!(*padfSrc != padfInNoData[iBand])) + *padfDst = padfOutNoData[iBand]; + else + *padfDst = data->LookupValue(iBand, *padfSrc); + ++padfSrc; + ++padfDst; + } + } + + return CE_None; +} + +/************************************************************************/ +/* DehazingData */ +/************************************************************************/ + +namespace +{ +/** Working structure for 'Dehazing' builtin function. */ +struct DehazingData +{ + static constexpr const char *const EXPECTED_SIGNATURE = "Dehazing"; + //! Signature (to make sure callback functions are called with the right argument) + const std::string m_osSignature = EXPECTED_SIGNATURE; + + //! Nodata value for gain dataset(s) + double m_dfGainNodata = std::numeric_limits::quiet_NaN(); + + //! Nodata value for offset dataset(s) + double m_dfOffsetNodata = std::numeric_limits::quiet_NaN(); + + //! Minimum clamping value. + double m_dfClampMin = std::numeric_limits::quiet_NaN(); + + //! Maximum clamping value. + double m_dfClampMax = std::numeric_limits::quiet_NaN(); + + //! Map from gain/offset dataset name to datasets + std::map> m_oDatasetMap{}; + + //! Vector of size nInBands that point to the raster band from which to read gains. + std::vector m_oGainBands{}; + + //! Vector of size nInBands that point to the raster band from which to read offsets. + std::vector m_oOffsetBands{}; + + //! Working buffer that contain gain values. + std::vector m_abyGainBuffer{}; + + //! Working buffer that contain offset values. + std::vector m_abyOffsetBuffer{}; +}; +} // namespace + +/************************************************************************/ +/* CheckAllBands() */ +/************************************************************************/ + +/** Return true if the key of oMap is the sequence of all integers between + * 0 and nExpectedBandCount-1. + */ +template +static bool CheckAllBands(const std::map &oMap, int nExpectedBandCount) +{ + int iExpected = 0; + for (const auto &kv : oMap) + { + if (kv.first != iExpected) + return false; + ++iExpected; + } + return iExpected == nExpectedBandCount; +} + +/************************************************************************/ +/* DehazingInit() */ +/************************************************************************/ + +/** Init function for 'Dehazing' builtin function. */ +static CPLErr DehazingInit(const char * /*pszFuncName*/, void * /*pUserData*/, + CSLConstList papszFunctionArgs, int nInBands, + GDALDataType eInDT, double *padfInNoData, + int *pnOutBands, GDALDataType *peOutDT, + double **ppadfOutNoData, const char *pszVRTPath, + VRTPDWorkingDataPtr *ppWorkingData) +{ + CPLAssert(eInDT == GDT_Float64); + + const bool bIsFinalStep = *pnOutBands != 0; + *peOutDT = eInDT; + *ppWorkingData = nullptr; + + if (!bIsFinalStep) + { + *pnOutBands = nInBands; + } + + auto data = std::make_unique(); + + bool bNodataSpecified = false; + double dfNoData = std::numeric_limits::quiet_NaN(); + + bool bGainNodataSpecified = false; + bool bOffsetNodataSpecified = false; + + std::map oGainDatasetNameMap; + std::map oGainDatasetBandMap; + + std::map oOffsetDatasetNameMap; + std::map oOffsetDatasetBandMap; + + bool bRelativeFilenames = false; + + for (const auto &[pszKey, pszValue] : + cpl::IterateNameValue(papszFunctionArgs)) + { + if (EQUAL(pszKey, "relative_filenames")) + { + bRelativeFilenames = CPLTestBool(pszValue); + } + else if (EQUAL(pszKey, "nodata")) + { + bNodataSpecified = true; + dfNoData = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "gain_nodata")) + { + bGainNodataSpecified = true; + data->m_dfGainNodata = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "offset_nodata")) + { + bOffsetNodataSpecified = true; + data->m_dfOffsetNodata = CPLAtof(pszValue); + } + else if (STARTS_WITH_CI(pszKey, "gain_dataset_filename_")) + { + const int nBand = atoi(pszKey + strlen("gain_dataset_filename_")); + if (nBand <= 0 || nBand > nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + oGainDatasetNameMap[nBand - 1] = pszValue; + } + else if (STARTS_WITH_CI(pszKey, "gain_dataset_band_")) + { + const int nBand = atoi(pszKey + strlen("gain_dataset_band_")); + if (nBand <= 0 || nBand > nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + oGainDatasetBandMap[nBand - 1] = atoi(pszValue); + } + else if (STARTS_WITH_CI(pszKey, "offset_dataset_filename_")) + { + const int nBand = atoi(pszKey + strlen("offset_dataset_filename_")); + if (nBand <= 0 || nBand > nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + oOffsetDatasetNameMap[nBand - 1] = pszValue; + } + else if (STARTS_WITH_CI(pszKey, "offset_dataset_band_")) + { + const int nBand = atoi(pszKey + strlen("offset_dataset_band_")); + if (nBand <= 0 || nBand > nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + oOffsetDatasetBandMap[nBand - 1] = atoi(pszValue); + } + else if (EQUAL(pszKey, "min")) + { + data->m_dfClampMin = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "max")) + { + data->m_dfClampMax = CPLAtof(pszValue); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Unrecognized argument name %s. Ignored", pszKey); + } + } + + if (!CheckAllBands(oGainDatasetNameMap, nInBands)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing gain_dataset_filename_XX element(s)"); + return CE_Failure; + } + if (!CheckAllBands(oGainDatasetBandMap, nInBands)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing gain_dataset_band_XX element(s)"); + return CE_Failure; + } + if (!CheckAllBands(oOffsetDatasetNameMap, nInBands)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing offset_dataset_filename_XX element(s)"); + return CE_Failure; + } + if (!CheckAllBands(oOffsetDatasetBandMap, nInBands)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing offset_dataset_band_XX element(s)"); + return CE_Failure; + } + + data->m_oGainBands.resize(nInBands); + data->m_oOffsetBands.resize(nInBands); + + constexpr int IDX_GAIN = 0; + constexpr int IDX_OFFSET = 1; + for (int i : {IDX_GAIN, IDX_OFFSET}) + { + const auto &oMapNames = + (i == IDX_GAIN) ? oGainDatasetNameMap : oOffsetDatasetNameMap; + const auto &oMapBands = + (i == IDX_GAIN) ? oGainDatasetBandMap : oOffsetDatasetBandMap; + for (const auto &kv : oMapNames) + { + const int nInBandIdx = kv.first; + const auto osFilename = VRTDataset::BuildSourceFilename( + kv.second.c_str(), pszVRTPath, bRelativeFilenames); + auto oIter = data->m_oDatasetMap.find(osFilename); + if (oIter == data->m_oDatasetMap.end()) + { + auto poDS = std::unique_ptr(GDALDataset::Open( + osFilename.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, + nullptr, nullptr, nullptr)); + if (!poDS) + return CE_Failure; + double adfAuxGT[6]; + if (poDS->GetGeoTransform(adfAuxGT) != CE_None) + { + CPLError(CE_Failure, CPLE_AppDefined, + "%s lacks a geotransform", osFilename.c_str()); + return CE_Failure; + } + oIter = data->m_oDatasetMap + .insert(std::pair(osFilename, std::move(poDS))) + .first; + } + auto poDS = oIter->second.get(); + const auto oIterBand = oMapBands.find(nInBandIdx); + CPLAssert(oIterBand != oMapBands.end()); + const int nAuxBand = oIterBand->second; + if (nAuxBand <= 0 || nAuxBand > poDS->GetRasterCount()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band number (%d) for a %s dataset", nAuxBand, + (i == IDX_GAIN) ? "gain" : "offset"); + return CE_Failure; + } + auto poAuxBand = poDS->GetRasterBand(nAuxBand); + int bAuxBandHasNoData = false; + const double dfAuxNoData = + poAuxBand->GetNoDataValue(&bAuxBandHasNoData); + if (i == IDX_GAIN) + { + data->m_oGainBands[nInBandIdx] = poAuxBand; + if (!bGainNodataSpecified && bAuxBandHasNoData) + data->m_dfGainNodata = dfAuxNoData; + } + else + { + data->m_oOffsetBands[nInBandIdx] = poAuxBand; + if (!bOffsetNodataSpecified && bAuxBandHasNoData) + data->m_dfOffsetNodata = dfAuxNoData; + } + } + } + + SetOutputValuesForInNoDataAndOutNoData( + nInBands, padfInNoData, pnOutBands, ppadfOutNoData, bNodataSpecified, + dfNoData, bNodataSpecified, dfNoData, bIsFinalStep); + + *ppWorkingData = data.release(); + return CE_None; +} + +/************************************************************************/ +/* DehazingFree() */ +/************************************************************************/ + +/** Free function for 'Dehazing' builtin function. */ +static void DehazingFree(const char * /*pszFuncName*/, void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData) +{ + DehazingData *data = static_cast(pWorkingData); + CPLAssert(data->m_osSignature == DehazingData::EXPECTED_SIGNATURE); + CPL_IGNORE_RET_VAL(data->m_osSignature); + delete data; +} + +/************************************************************************/ +/* LoadAuxData() */ +/************************************************************************/ + +// Load auxiliary corresponding offset, gain or trimming data. +static bool LoadAuxData(double dfULX, double dfULY, double dfLRX, double dfLRY, + size_t nElts, int nBufXSize, int nBufYSize, + const char *pszAuxType, GDALRasterBand *poAuxBand, + std::vector &abyBuffer) +{ + double adfAuxGT[6]; + double adfAuxInvGT[6]; + + // Compute pixel/line coordinates from the georeferenced extent + CPL_IGNORE_RET_VAL(poAuxBand->GetDataset()->GetGeoTransform( + adfAuxGT)); // return code already tested + CPL_IGNORE_RET_VAL(GDALInvGeoTransform(adfAuxGT, adfAuxInvGT)); + const double dfULPixel = + adfAuxInvGT[0] + adfAuxInvGT[1] * dfULX + adfAuxInvGT[2] * dfULY; + const double dfULLine = + adfAuxInvGT[3] + adfAuxInvGT[4] * dfULX + adfAuxInvGT[5] * dfULY; + const double dfLRPixel = + adfAuxInvGT[0] + adfAuxInvGT[1] * dfLRX + adfAuxInvGT[2] * dfLRY; + const double dfLRLine = + adfAuxInvGT[3] + adfAuxInvGT[4] * dfLRX + adfAuxInvGT[5] * dfLRY; + if (dfULPixel >= dfLRPixel || dfULLine >= dfLRLine) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Unexpected computed %s pixel/line", pszAuxType); + return false; + } + if (dfULPixel < -1 || dfLRPixel > poAuxBand->GetXSize() || dfULLine < -1 || + dfLRLine > poAuxBand->GetYSize()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Unexpected computed %s pixel/line", pszAuxType); + return false; + } + + const int nAuxXOff = std::max(0, static_cast(std::round(dfULPixel))); + const int nAuxYOff = std::max(0, static_cast(std::round(dfULLine))); + const int nAuxX2Off = std::min(poAuxBand->GetXSize(), + static_cast(std::round(dfLRPixel))); + const int nAuxY2Off = + std::min(poAuxBand->GetYSize(), static_cast(std::round(dfLRLine))); + + try + { + abyBuffer.resize(nElts * sizeof(float)); + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory allocating working buffer"); + return false; + } + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + sExtraArg.bFloatingPointWindowValidity = true; + CPL_IGNORE_RET_VAL(sExtraArg.eResampleAlg); + sExtraArg.eResampleAlg = GRIORA_Bilinear; + sExtraArg.dfXOff = std::max(0.0, dfULPixel); + sExtraArg.dfYOff = std::max(0.0, dfULLine); + sExtraArg.dfXSize = std::min(poAuxBand->GetXSize(), dfLRPixel) - + std::max(0.0, dfULPixel); + sExtraArg.dfYSize = std::min(poAuxBand->GetYSize(), dfLRLine) - + std::max(0.0, dfULLine); + return (poAuxBand->RasterIO( + GF_Read, nAuxXOff, nAuxYOff, std::max(1, nAuxX2Off - nAuxXOff), + std::max(1, nAuxY2Off - nAuxYOff), abyBuffer.data(), nBufXSize, + nBufYSize, GDT_Float32, 0, 0, &sExtraArg) == CE_None); +} + +/************************************************************************/ +/* DehazingProcess() */ +/************************************************************************/ + +/** Processing function for 'Dehazing' builtin function. */ +static CPLErr DehazingProcess( + const char * /*pszFuncName*/, void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData, CSLConstList /* papszFunctionArgs*/, + int nBufXSize, int nBufYSize, const void *pInBuffer, size_t nInBufferSize, + GDALDataType eInDT, int nInBands, const double *CPL_RESTRICT padfInNoData, + void *pOutBuffer, size_t nOutBufferSize, GDALDataType eOutDT, int nOutBands, + const double *CPL_RESTRICT padfOutNoData, double dfSrcXOff, + double dfSrcYOff, double dfSrcXSize, double dfSrcYSize, + const double adfSrcGT[], const char * /* pszVRTPath */, + CSLConstList /*papszExtra*/) +{ + const size_t nElts = static_cast(nBufXSize) * nBufYSize; + + CPL_IGNORE_RET_VAL(eInDT); + CPLAssert(eInDT == GDT_Float64); + CPL_IGNORE_RET_VAL(eOutDT); + CPLAssert(eOutDT == GDT_Float64); + CPL_IGNORE_RET_VAL(nInBufferSize); + CPLAssert(nInBufferSize == nElts * nInBands * sizeof(double)); + CPL_IGNORE_RET_VAL(nOutBufferSize); + CPLAssert(nOutBufferSize == nElts * nOutBands * sizeof(double)); + CPLAssert(nInBands == nOutBands); + CPL_IGNORE_RET_VAL(nOutBands); + + DehazingData *data = static_cast(pWorkingData); + CPLAssert(data->m_osSignature == DehazingData::EXPECTED_SIGNATURE); + const double *CPL_RESTRICT padfSrc = static_cast(pInBuffer); + double *CPL_RESTRICT padfDst = static_cast(pOutBuffer); + + // Compute georeferenced extent of input region + const double dfULX = + adfSrcGT[0] + adfSrcGT[1] * dfSrcXOff + adfSrcGT[2] * dfSrcYOff; + const double dfULY = + adfSrcGT[3] + adfSrcGT[4] * dfSrcXOff + adfSrcGT[5] * dfSrcYOff; + const double dfLRX = adfSrcGT[0] + adfSrcGT[1] * (dfSrcXOff + dfSrcXSize) + + adfSrcGT[2] * (dfSrcYOff + dfSrcYSize); + const double dfLRY = adfSrcGT[3] + adfSrcGT[4] * (dfSrcXOff + dfSrcXSize) + + adfSrcGT[5] * (dfSrcYOff + dfSrcYSize); + + auto &abyOffsetBuffer = data->m_abyGainBuffer; + auto &abyGainBuffer = data->m_abyOffsetBuffer; + + for (int iBand = 0; iBand < nInBands; ++iBand) + { + if (!LoadAuxData(dfULX, dfULY, dfLRX, dfLRY, nElts, nBufXSize, + nBufYSize, "gain", data->m_oGainBands[iBand], + abyGainBuffer) || + !LoadAuxData(dfULX, dfULY, dfLRX, dfLRY, nElts, nBufXSize, + nBufYSize, "offset", data->m_oOffsetBands[iBand], + abyOffsetBuffer)) + { + return CE_Failure; + } + + const double *CPL_RESTRICT padfSrcThisBand = padfSrc + iBand; + double *CPL_RESTRICT padfDstThisBand = padfDst + iBand; + const float *pafGain = + reinterpret_cast(abyGainBuffer.data()); + const float *pafOffset = + reinterpret_cast(abyOffsetBuffer.data()); + const double dfSrcNodata = padfInNoData[iBand]; + const double dfDstNodata = padfOutNoData[iBand]; + const double dfGainNodata = data->m_dfGainNodata; + const double dfOffsetNodata = data->m_dfOffsetNodata; + const double dfClampMin = data->m_dfClampMin; + const double dfClampMax = data->m_dfClampMax; + for (size_t i = 0; i < nElts; ++i) + { + const double dfSrcVal = *padfSrcThisBand; + // written this way to work with a NaN value + if (!(dfSrcVal != dfSrcNodata)) + { + *padfDstThisBand = dfDstNodata; + } + else + { + const double dfGain = pafGain[i]; + const double dfOffset = pafOffset[i]; + if (!(dfGain != dfGainNodata) || !(dfOffset != dfOffsetNodata)) + { + *padfDstThisBand = dfDstNodata; + } + else + { + double dfDehazed = dfSrcVal * dfGain - dfOffset; + if (dfDehazed < dfClampMin) + dfDehazed = dfClampMin; + if (dfDehazed > dfClampMax) + dfDehazed = dfClampMax; + + *padfDstThisBand = dfDehazed; + } + } + padfSrcThisBand += nInBands; + padfDstThisBand += nInBands; + } + } + + return CE_None; +} + +/************************************************************************/ +/* TrimmingData */ +/************************************************************************/ + +namespace +{ +/** Working structure for 'Trimming' builtin function. */ +struct TrimmingData +{ + static constexpr const char *const EXPECTED_SIGNATURE = "Trimming"; + //! Signature (to make sure callback functions are called with the right argument) + const std::string m_osSignature = EXPECTED_SIGNATURE; + + //! Nodata value for trimming dataset + double m_dfTrimmingNodata = std::numeric_limits::quiet_NaN(); + + //! Maximum saturating RGB output value. + double m_dfTopRGB = 0; + + //! Maximum threshold beyond which we give up saturation + double m_dfToneCeil = 0; + + //! Margin to allow for dynamics in brighest areas (in [0,1] range) + double m_dfTopMargin = 0; + + //! Index (zero-based) of input/output red band. + int m_nRedBand = 1 - 1; + + //! Index (zero-based) of input/output green band. + int m_nGreenBand = 2 - 1; + + //! Index (zero-based) of input/output blue band. + int m_nBlueBand = 3 - 1; + + //! Trimming dataset + std::unique_ptr m_poTrimmingDS{}; + + //! Trimming raster band. + GDALRasterBand *m_poTrimmingBand = nullptr; + + //! Working buffer that contain trimming values. + std::vector m_abyTrimmingBuffer{}; +}; +} // namespace + +/************************************************************************/ +/* TrimmingInit() */ +/************************************************************************/ + +/** Init function for 'Trimming' builtin function. */ +static CPLErr TrimmingInit(const char * /*pszFuncName*/, void * /*pUserData*/, + CSLConstList papszFunctionArgs, int nInBands, + GDALDataType eInDT, double *padfInNoData, + int *pnOutBands, GDALDataType *peOutDT, + double **ppadfOutNoData, const char *pszVRTPath, + VRTPDWorkingDataPtr *ppWorkingData) +{ + CPLAssert(eInDT == GDT_Float64); + + const bool bIsFinalStep = *pnOutBands != 0; + *peOutDT = eInDT; + *ppWorkingData = nullptr; + + if (!bIsFinalStep) + { + *pnOutBands = nInBands; + } + + auto data = std::make_unique(); + + bool bNodataSpecified = false; + double dfNoData = std::numeric_limits::quiet_NaN(); + std::string osTrimmingFilename; + bool bTrimmingNodataSpecified = false; + bool bRelativeFilenames = false; + + for (const auto &[pszKey, pszValue] : + cpl::IterateNameValue(papszFunctionArgs)) + { + if (EQUAL(pszKey, "relative_filenames")) + { + bRelativeFilenames = CPLTestBool(pszValue); + } + else if (EQUAL(pszKey, "nodata")) + { + bNodataSpecified = true; + dfNoData = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "trimming_nodata")) + { + bTrimmingNodataSpecified = true; + data->m_dfTrimmingNodata = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "trimming_dataset_filename")) + { + osTrimmingFilename = pszValue; + } + else if (EQUAL(pszKey, "red_band")) + { + const int nBand = atoi(pszValue) - 1; + if (nBand < 0 || nBand >= nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + data->m_nRedBand = nBand; + } + else if (EQUAL(pszKey, "green_band")) + { + const int nBand = atoi(pszValue) - 1; + if (nBand < 0 || nBand >= nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + data->m_nGreenBand = nBand; + } + else if (EQUAL(pszKey, "blue_band")) + { + const int nBand = atoi(pszValue) - 1; + if (nBand < 0 || nBand >= nInBands) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid band in argument '%s'", pszKey); + return CE_Failure; + } + data->m_nBlueBand = nBand; + } + else if (EQUAL(pszKey, "top_rgb")) + { + data->m_dfTopRGB = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "tone_ceil")) + { + data->m_dfToneCeil = CPLAtof(pszValue); + } + else if (EQUAL(pszKey, "top_margin")) + { + data->m_dfTopMargin = CPLAtof(pszValue); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Unrecognized argument name %s. Ignored", pszKey); + } + } + + if (data->m_nRedBand == data->m_nGreenBand || + data->m_nRedBand == data->m_nBlueBand || + data->m_nGreenBand == data->m_nBlueBand) + { + CPLError( + CE_Failure, CPLE_NotSupported, + "red_band, green_band and blue_band must have distinct values"); + return CE_Failure; + } + + const auto osFilename = VRTDataset::BuildSourceFilename( + osTrimmingFilename.c_str(), pszVRTPath, bRelativeFilenames); + data->m_poTrimmingDS.reset(GDALDataset::Open( + osFilename.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, nullptr, + nullptr, nullptr)); + if (!data->m_poTrimmingDS) + return CE_Failure; + if (data->m_poTrimmingDS->GetRasterCount() != 1) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Trimming dataset should have a single band"); + return CE_Failure; + } + data->m_poTrimmingBand = data->m_poTrimmingDS->GetRasterBand(1); + + double adfAuxGT[6]; + if (data->m_poTrimmingDS->GetGeoTransform(adfAuxGT) != CE_None) + { + CPLError(CE_Failure, CPLE_AppDefined, "%s lacks a geotransform", + osFilename.c_str()); + return CE_Failure; + } + int bAuxBandHasNoData = false; + const double dfAuxNoData = + data->m_poTrimmingBand->GetNoDataValue(&bAuxBandHasNoData); + if (!bTrimmingNodataSpecified && bAuxBandHasNoData) + data->m_dfTrimmingNodata = dfAuxNoData; + + SetOutputValuesForInNoDataAndOutNoData( + nInBands, padfInNoData, pnOutBands, ppadfOutNoData, bNodataSpecified, + dfNoData, bNodataSpecified, dfNoData, bIsFinalStep); + + *ppWorkingData = data.release(); + return CE_None; +} + +/************************************************************************/ +/* TrimmingFree() */ +/************************************************************************/ + +/** Free function for 'Trimming' builtin function. */ +static void TrimmingFree(const char * /*pszFuncName*/, void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData) +{ + TrimmingData *data = static_cast(pWorkingData); + CPLAssert(data->m_osSignature == TrimmingData::EXPECTED_SIGNATURE); + CPL_IGNORE_RET_VAL(data->m_osSignature); + delete data; +} + +/************************************************************************/ +/* TrimmingProcess() */ +/************************************************************************/ + +/** Processing function for 'Trimming' builtin function. */ +static CPLErr TrimmingProcess( + const char * /*pszFuncName*/, void * /*pUserData*/, + VRTPDWorkingDataPtr pWorkingData, CSLConstList /* papszFunctionArgs*/, + int nBufXSize, int nBufYSize, const void *pInBuffer, size_t nInBufferSize, + GDALDataType eInDT, int nInBands, const double *CPL_RESTRICT padfInNoData, + void *pOutBuffer, size_t nOutBufferSize, GDALDataType eOutDT, int nOutBands, + const double *CPL_RESTRICT padfOutNoData, double dfSrcXOff, + double dfSrcYOff, double dfSrcXSize, double dfSrcYSize, + const double adfSrcGT[], const char * /* pszVRTPath */, + CSLConstList /*papszExtra*/) +{ + const size_t nElts = static_cast(nBufXSize) * nBufYSize; + + CPL_IGNORE_RET_VAL(eInDT); + CPLAssert(eInDT == GDT_Float64); + CPL_IGNORE_RET_VAL(eOutDT); + CPLAssert(eOutDT == GDT_Float64); + CPL_IGNORE_RET_VAL(nInBufferSize); + CPLAssert(nInBufferSize == nElts * nInBands * sizeof(double)); + CPL_IGNORE_RET_VAL(nOutBufferSize); + CPLAssert(nOutBufferSize == nElts * nOutBands * sizeof(double)); + CPLAssert(nInBands == nOutBands); + CPL_IGNORE_RET_VAL(nOutBands); + + TrimmingData *data = static_cast(pWorkingData); + CPLAssert(data->m_osSignature == TrimmingData::EXPECTED_SIGNATURE); + const double *CPL_RESTRICT padfSrc = static_cast(pInBuffer); + double *CPL_RESTRICT padfDst = static_cast(pOutBuffer); + + // Compute georeferenced extent of input region + const double dfULX = + adfSrcGT[0] + adfSrcGT[1] * dfSrcXOff + adfSrcGT[2] * dfSrcYOff; + const double dfULY = + adfSrcGT[3] + adfSrcGT[4] * dfSrcXOff + adfSrcGT[5] * dfSrcYOff; + const double dfLRX = adfSrcGT[0] + adfSrcGT[1] * (dfSrcXOff + dfSrcXSize) + + adfSrcGT[2] * (dfSrcYOff + dfSrcYSize); + const double dfLRY = adfSrcGT[3] + adfSrcGT[4] * (dfSrcXOff + dfSrcXSize) + + adfSrcGT[5] * (dfSrcYOff + dfSrcYSize); + + if (!LoadAuxData(dfULX, dfULY, dfLRX, dfLRY, nElts, nBufXSize, nBufYSize, + "trimming", data->m_poTrimmingBand, + data->m_abyTrimmingBuffer)) + { + return CE_Failure; + } + + const float *pafTrimming = + reinterpret_cast(data->m_abyTrimmingBuffer.data()); + const int nRedBand = data->m_nRedBand; + const int nGreenBand = data->m_nGreenBand; + const int nBlueBand = data->m_nBlueBand; + const double dfTopMargin = data->m_dfTopMargin; + const double dfTopRGB = data->m_dfTopRGB; + const double dfToneCeil = data->m_dfToneCeil; +#if !defined(trimming_non_optimized_version) + const double dfInvToneCeil = 1.0 / dfToneCeil; +#endif + const bool bRGBBandsAreFirst = + std::max(std::max(nRedBand, nGreenBand), nBlueBand) <= 2; + const double dfNoDataTrimming = data->m_dfTrimmingNodata; + const double dfNoDataRed = padfInNoData[nRedBand]; + const double dfNoDataGreen = padfInNoData[nGreenBand]; + const double dfNoDataBlue = padfInNoData[nBlueBand]; + for (size_t i = 0; i < nElts; ++i) + { + // Extract local saturation value from trimming image + const double dfLocalMaxRGB = pafTrimming[i]; + const double dfReducedRGB = + std::min((1.0 - dfTopMargin) * dfTopRGB / dfLocalMaxRGB, 1.0); + + const double dfRed = padfSrc[nRedBand]; + const double dfGreen = padfSrc[nGreenBand]; + const double dfBlue = padfSrc[nBlueBand]; + bool bNoDataPixel = false; + if ((dfLocalMaxRGB != dfNoDataTrimming) && (dfRed != dfNoDataRed) && + (dfGreen != dfNoDataGreen) && (dfBlue != dfNoDataBlue)) + { + // RGB bands specific process + const double dfMaxRGB = std::max(std::max(dfRed, dfGreen), dfBlue); +#if !defined(trimming_non_optimized_version) + const double dfRedTimesToneRed = std::min(dfRed, dfToneCeil); + const double dfGreenTimesToneGreen = std::min(dfGreen, dfToneCeil); + const double dfBlueTimesToneBlue = std::min(dfBlue, dfToneCeil); + const double dfInvToneMaxRGB = + std::max(dfMaxRGB * dfInvToneCeil, 1.0); + const double dfReducedRGBTimesInvToneMaxRGB = + dfReducedRGB * dfInvToneMaxRGB; + padfDst[nRedBand] = std::min( + dfRedTimesToneRed * dfReducedRGBTimesInvToneMaxRGB, dfTopRGB); + padfDst[nGreenBand] = + std::min(dfGreenTimesToneGreen * dfReducedRGBTimesInvToneMaxRGB, + dfTopRGB); + padfDst[nBlueBand] = std::min( + dfBlueTimesToneBlue * dfReducedRGBTimesInvToneMaxRGB, dfTopRGB); +#else + // Original formulas. Slightly less optimized than the above ones. + const double dfToneMaxRGB = std::min(dfToneCeil / dfMaxRGB, 1.0); + const double dfToneRed = std::min(dfToneCeil / dfRed, 1.0); + const double dfToneGreen = std::min(dfToneCeil / dfGreen, 1.0); + const double dfToneBlue = std::min(dfToneCeil / dfBlue, 1.0); + padfDst[nRedBand] = std::min( + dfReducedRGB * dfRed * dfToneRed / dfToneMaxRGB, dfTopRGB); + padfDst[nGreenBand] = std::min( + dfReducedRGB * dfGreen * dfToneGreen / dfToneMaxRGB, dfTopRGB); + padfDst[nBlueBand] = std::min( + dfReducedRGB * dfBlue * dfToneBlue / dfToneMaxRGB, dfTopRGB); +#endif + + // Other bands processing (NIR, ...): only apply RGB reduction factor + if (bRGBBandsAreFirst) + { + // optimization + for (int iBand = 3; iBand < nInBands; ++iBand) + { + if (padfSrc[iBand] != padfInNoData[iBand]) + { + padfDst[iBand] = dfReducedRGB * padfSrc[iBand]; + } + else + { + bNoDataPixel = true; + break; + } + } + } + else + { + for (int iBand = 0; iBand < nInBands; ++iBand) + { + if (iBand != nRedBand && iBand != nGreenBand && + iBand != nBlueBand) + { + if (padfSrc[iBand] != padfInNoData[iBand]) + { + padfDst[iBand] = dfReducedRGB * padfSrc[iBand]; + } + else + { + bNoDataPixel = true; + break; + } + } + } + } + } + else + { + bNoDataPixel = true; + } + if (bNoDataPixel) + { + for (int iBand = 0; iBand < nInBands; ++iBand) + { + padfDst[iBand] = padfOutNoData[iBand]; + } + } + + padfSrc += nInBands; + padfDst += nInBands; + } + + return CE_None; +} + +/************************************************************************/ +/* GDALVRTRegisterDefaultProcessedDatasetFuncs() */ +/************************************************************************/ + +/** Register builtin functions that can be used in a VRTProcessedDataset. + */ +void GDALVRTRegisterDefaultProcessedDatasetFuncs() +{ + GDALVRTRegisterProcessedDatasetFunc( + "BandAffineCombination", nullptr, + "" + " " + " " + " " + " " + " " + " " + " " + "", + GDT_Float64, nullptr, 0, nullptr, 0, BandAffineCombinationInit, + BandAffineCombinationFree, BandAffineCombinationProcess, nullptr); + + GDALVRTRegisterProcessedDatasetFunc( + "LUT", nullptr, + "" + " " + " " + " " + "", + GDT_Float64, nullptr, 0, nullptr, 0, LUTInit, LUTFree, LUTProcess, + nullptr); + + GDALVRTRegisterProcessedDatasetFunc( + "Dehazing", nullptr, + "" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "", + GDT_Float64, nullptr, 0, nullptr, 0, DehazingInit, DehazingFree, + DehazingProcess, nullptr); + + GDALVRTRegisterProcessedDatasetFunc( + "Trimming", nullptr, + "" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "", + GDT_Float64, nullptr, 0, nullptr, 0, TrimmingInit, TrimmingFree, + TrimmingProcess, nullptr); +} diff --git a/frmts/vrt/vrtsources.cpp b/frmts/vrt/vrtsources.cpp index b10ce6d44123..562d5e8c77ad 100644 --- a/frmts/vrt/vrtsources.cpp +++ b/frmts/vrt/vrtsources.cpp @@ -285,11 +285,6 @@ void VRTSimpleSource::GetDstWindow(double &dfDstXOff, double &dfDstYOff, /* SerializeToXML() */ /************************************************************************/ -static const char *const apszSpecialSyntax[] = { - "NITF_IM:{ANY}:{FILENAME}", "PDF:{ANY}:{FILENAME}", - "RASTERLITE:{FILENAME},{ANY}", "TILEDB:\"{FILENAME}\":{ANY}", - "TILEDB:{FILENAME}:{ANY}"}; - static bool IsSlowSource(const char *pszSrcName) { return strstr(pszSrcName, "/vsicurl/http") != nullptr || @@ -350,11 +345,8 @@ CPLXMLNode *VRTSimpleSource::SerializeToXML(const char *pszVRTPath) } else { - for (size_t i = 0; - i < sizeof(apszSpecialSyntax) / sizeof(apszSpecialSyntax[0]); - ++i) + for (const char *pszSyntax : VRTDataset::apszSpecialSyntax) { - const char *const pszSyntax = apszSpecialSyntax[i]; CPLString osPrefix(pszSyntax); osPrefix.resize(strchr(pszSyntax, ':') - pszSyntax + 1); if (pszSyntax[osPrefix.size()] == '"') @@ -552,86 +544,8 @@ VRTSimpleSource::XMLInit(const CPLXMLNode *psSrc, const char *pszVRTPath, m_nExplicitSharedStatus = CPLTestBool(pszShared); } - if (pszVRTPath != nullptr && m_bRelativeToVRTOri) - { - // Try subdatasetinfo API first - // Note: this will become the only branch when subdatasetinfo will become - // available for NITF_IM, RASTERLITE and TILEDB - const auto oSubDSInfo{GDALGetSubdatasetInfo(pszFilename)}; - if (oSubDSInfo && !oSubDSInfo->GetPathComponent().empty()) - { - auto path{oSubDSInfo->GetPathComponent()}; - m_osSrcDSName = oSubDSInfo->ModifyPathComponent( - CPLProjectRelativeFilename(pszVRTPath, path.c_str())); - GDALDestroySubdatasetInfo(oSubDSInfo); - } - else - { - bool bDone = false; - for (size_t i = 0; - i < sizeof(apszSpecialSyntax) / sizeof(apszSpecialSyntax[0]); - ++i) - { - const char *pszSyntax = apszSpecialSyntax[i]; - CPLString osPrefix(pszSyntax); - osPrefix.resize(strchr(pszSyntax, ':') - pszSyntax + 1); - if (pszSyntax[osPrefix.size()] == '"') - osPrefix += '"'; - if (EQUALN(pszFilename, osPrefix, osPrefix.size())) - { - if (STARTS_WITH_CI(pszSyntax + osPrefix.size(), "{ANY}")) - { - const char *pszLastPart = strrchr(pszFilename, ':') + 1; - // CSV:z:/foo.xyz - if ((pszLastPart[0] == '/' || pszLastPart[0] == '\\') && - pszLastPart - pszFilename >= 3 && - pszLastPart[-3] == ':') - { - pszLastPart -= 2; - } - CPLString osPrefixFilename = pszFilename; - osPrefixFilename.resize(pszLastPart - pszFilename); - m_osSrcDSName = - osPrefixFilename + - CPLProjectRelativeFilename(pszVRTPath, pszLastPart); - bDone = true; - } - else if (STARTS_WITH_CI(pszSyntax + osPrefix.size(), - "{FILENAME}")) - { - CPLString osFilename(pszFilename + osPrefix.size()); - size_t nPos = 0; - if (osFilename.size() >= 3 && osFilename[1] == ':' && - (osFilename[2] == '\\' || osFilename[2] == '/')) - nPos = 2; - nPos = osFilename.find( - pszSyntax[osPrefix.size() + strlen("{FILENAME}")], - nPos); - if (nPos != std::string::npos) - { - const CPLString osSuffix = osFilename.substr(nPos); - osFilename.resize(nPos); - m_osSrcDSName = osPrefix + - CPLProjectRelativeFilename( - pszVRTPath, osFilename) + - osSuffix; - bDone = true; - } - } - break; - } - } - if (!bDone) - { - m_osSrcDSName = - CPLProjectRelativeFilename(pszVRTPath, pszFilename); - } - } - } - else - { - m_osSrcDSName = pszFilename; - } + m_osSrcDSName = VRTDataset::BuildSourceFilename( + pszFilename, pszVRTPath, CPL_TO_BOOL(m_bRelativeToVRTOri)); const char *pszSourceBand = CPLGetXMLValue(psSrc, "SourceBand", "1"); m_bGetMaskBand = false; diff --git a/gcore/gdal.h b/gcore/gdal.h index 313e1362722d..9599f1d2b143 100644 --- a/gcore/gdal.h +++ b/gcore/gdal.h @@ -1640,6 +1640,115 @@ CPLErr CPL_DLL CPL_STDCALL GDALAddDerivedBandPixelFuncWithArgs( const char *pszName, GDALDerivedPixelFuncWithArgs pfnPixelFunc, const char *pszMetadata); +/** Generic pointer for the working structure of VRTProcessedDataset + * function. */ +typedef void *VRTPDWorkingDataPtr; + +/** Initialization function to pass to GDALVRTRegisterProcessedDatasetFunc. + * + * This initialization function is called for each step of a VRTProcessedDataset + * that uses the related algorithm. + * The initialization function returns the output data type, output band count + * and potentially initializes a working structure, typically parsing arguments. + * + * @param pszFuncName Function name. Must be unique and not null. + * @param pUserData User data. May be nullptr. Must remain valid during the + * lifetime of GDAL. + * @param papszFunctionArgs Function arguments as a list of key=value pairs. + * @param nInBands Number of input bands. + * @param eInDT Input data type. + * @param[in,out] padfInNoData Array of nInBands values for the input nodata + * value. The init function may also override them. + * @param[in,out] pnOutBands Pointer whose value must be set to the number of + * output bands. This will be set to 0 by the caller + * when calling the function, unless this is the + * final step, in which case it will be initialized + * with the number of expected output bands. + * @param[out] peOutDT Pointer whose value must be set to the output + * data type. + * @param[in,out] ppadfOutNoData Pointer to an array of *pnOutBands values + * for the output nodata value that the + * function must set. + * For non-final steps, *ppadfOutNoData + * will be nullptr and it is the responsibility + * of the function to CPLMalloc()'ate it. + * If this is the final step, it will be + * already allocated and initialized with the + * expected nodata values from the output + * dataset (if the init function need to + * reallocate it, it must use CPLRealloc()) + * @param pszVRTPath Directory of the VRT + * @param[out] ppWorkingData Pointer whose value must be set to a working + * structure, or nullptr. + * @return CE_None in case of success, error otherwise. + * @since GDAL 3.9 */ +typedef CPLErr (*GDALVRTProcessedDatasetFuncInit)( + const char *pszFuncName, void *pUserData, CSLConstList papszFunctionArgs, + int nInBands, GDALDataType eInDT, double *padfInNoData, int *pnOutBands, + GDALDataType *peOutDT, double **ppadfOutNoData, const char *pszVRTPath, + VRTPDWorkingDataPtr *ppWorkingData); + +/** Free function to pass to GDALVRTRegisterProcessedDatasetFunc. + * + * @param pszFuncName Function name. Must be unique and not null. + * @param pUserData User data. May be nullptr. Must remain valid during the + * lifetime of GDAL. + * @param pWorkingData Value of the *ppWorkingData output parameter of + * GDALVRTProcessedDatasetFuncInit. + * @since GDAL 3.9 + */ +typedef void (*GDALVRTProcessedDatasetFuncFree)( + const char *pszFuncName, void *pUserData, VRTPDWorkingDataPtr pWorkingData); + +/** Processing function to pass to GDALVRTRegisterProcessedDatasetFunc. + * @param pszFuncName Function name. Must be unique and not null. + * @param pUserData User data. May be nullptr. Must remain valid during the + * lifetime of GDAL. + * @param pWorkingData Value of the *ppWorkingData output parameter of + * GDALVRTProcessedDatasetFuncInit. + * @param papszFunctionArgs Function arguments as a list of key=value pairs. + * @param nBufXSize Width in pixels of pInBuffer and pOutBuffer + * @param nBufYSize Height in pixels of pInBuffer and pOutBuffer + * @param pInBuffer Input buffer. It is pixel-interleaved + * (i.e. R00,G00,B00,R01,G01,B01, etc.) + * @param nInBufferSize Size in bytes of pInBuffer + * @param eInDT Data type of pInBuffer + * @param nInBands Number of bands in pInBuffer. + * @param padfInNoData Input nodata values. + * @param pOutBuffer Output buffer. It is pixel-interleaved + * (i.e. R00,G00,B00,R01,G01,B01, etc.) + * @param nOutBufferSize Size in bytes of pOutBuffer + * @param eOutDT Data type of pOutBuffer + * @param nOutBands Number of bands in pOutBuffer. + * @param padfOutNoData Input nodata values. + * @param dfSrcXOff Source X coordinate in pixel of the top-left of the region + * @param dfSrcYOff Source Y coordinate in pixel of the top-left of the region + * @param dfSrcXSize Width in pixels of the region + * @param dfSrcYSize Height in pixels of the region + * @param adfSrcGT Source geotransform + * @param pszVRTPath Directory of the VRT + * @param papszExtra Extra arguments (unused for now) + * @since GDAL 3.9 + */ +typedef CPLErr (*GDALVRTProcessedDatasetFuncProcess)( + const char *pszFuncName, void *pUserData, VRTPDWorkingDataPtr pWorkingData, + CSLConstList papszFunctionArgs, int nBufXSize, int nBufYSize, + const void *pInBuffer, size_t nInBufferSize, GDALDataType eInDT, + int nInBands, const double *padfInNoData, void *pOutBuffer, + size_t nOutBufferSize, GDALDataType eOutDT, int nOutBands, + const double *padfOutNoData, double dfSrcXOff, double dfSrcYOff, + double dfSrcXSize, double dfSrcYSize, const double adfSrcGT[/*6*/], + const char *pszVRTPath, CSLConstList papszExtra); + +CPLErr CPL_DLL GDALVRTRegisterProcessedDatasetFunc( + const char *pszFuncName, void *pUserData, const char *pszXMLMetadata, + GDALDataType eRequestedInputDT, const GDALDataType *paeSupportedInputDT, + size_t nSupportedInputDTSize, const int *panSupportedInputBandCount, + size_t nSupportedInputBandCountSize, + GDALVRTProcessedDatasetFuncInit pfnInit, + GDALVRTProcessedDatasetFuncFree pfnFree, + GDALVRTProcessedDatasetFuncProcess pfnProcess, CSLConstList papszOptions); + GDALRasterBandH CPL_DLL CPL_STDCALL GDALGetMaskBand(GDALRasterBandH hBand); int CPL_DLL CPL_STDCALL GDALGetMaskFlags(GDALRasterBandH hBand); CPLErr CPL_DLL CPL_STDCALL GDALCreateMaskBand(GDALRasterBandH hBand,