VRT: add a new mode to apply chained processing steps that apply to several bands at the same time

.github/workflows/cifuzz.yml

@@ -15,6 +15,7 @@ permissions:
 jobs:
   Fuzzing:
     runs-on: ubuntu-latest
+    if: github.repository == 'OSGeo/gdal'
     steps:
     - name: Build Fuzzers
       id: build

.github/workflows/code_checks.yml

@@ -82,7 +82,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
       - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
 
   doxygen:
@@ -127,7 +127,7 @@ jobs:
     - name: Checkout
       uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
     - name: Set up Python
-      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+      uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
       with:
         python-version: 3.8
     - name: Install lint tool

.github/workflows/codeql.yml

@@ -43,22 +43,6 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-        config: |
-          query-filters:
-            - exclude:
-                id: cpp/non-https-url
-
     - name: Install dependencies
       run: |
           sudo apt-get update
@@ -104,55 +88,34 @@ jobs:
             libzstd-dev \
             unixodbc-dev
 
-    # cache the .ccache directory
-    # key it on the runner os, build type, deps, and arch
-    # It's especially important to include arch in the key because we
-    # may get runtime errors with -mavx2 from objects built on a
-    # different architecture.
-    - name: Restore build cache
+    - name: Configure
       if: matrix.language == 'c-cpp'
-      id: restore-cache
-      uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      run: |
+          mkdir build
+          (cd build && cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DGDAL_USE_LERC_INTERNAL=OFF)
+
+    # Initializes the CodeQL tools for scanning.
+    # We do that after running CMake to avoid CodeQL to trigger during CMake time,
+    # in particular during HDF5 detection which is terribly slow (https://github.com/OSGeo/gdal/issues/9549)
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9
       with:
-        path: ${{ github.workspace }}/.ccache
-        key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }}
-        restore-keys: |
-          ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}
-          ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
 
-    - name: Configure ccache
-      if: matrix.language == 'c-cpp'
-      run: |
-          echo CCACHE_BASEDIR=${{ github.workspace }} >> ${GITHUB_ENV}
-          echo CCACHE_DIR=${{ github.workspace }}/.ccache >> ${GITHUB_ENV}
-          echo CCACHE_MAXSIZE=250M >> ${GITHUB_ENV}
-          ccache -z
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+        config: |
+          query-filters:
+            - exclude:
+                id: cpp/non-https-url
 
     - name: Build
       if: matrix.language == 'c-cpp'
       run: |
-          mkdir build
-          cd build
-          # LD_PRELOAD is initially set to something like /opt/hostedtoolcache/CodeQL/2.16.5/x64/codeql/tools/linux64/lib64trace.so
-          # To avoid CodeQL to trigger during CMake time, in particular during HDF5 detection which is terribly slow (https://github.com/OSGeo/gdal/issues/9549), we temporarily rename that file
-          export LD_PRELOAD_RESOLVED=$(env | grep LD_PRELOAD | sed "s/LD_PRELOAD=//")
-          echo "$LD_PRELOAD_RESOLVED"
-          sudo mv "$LD_PRELOAD_RESOLVED" "$LD_PRELOAD_RESOLVED.disabled"
-          cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSE_CCACHE=YES -DGDAL_USE_LERC_INTERNAL=OFF
-          sudo mv "$LD_PRELOAD_RESOLVED.disabled" "$LD_PRELOAD_RESOLVED"
-          make -j$(nproc)
-
-    - name: Summarize ccache
-      if: matrix.language == 'c-cpp'
-      run: |
-        ccache -s
-
-    - name: Save build cache
-      if: matrix.language == 'c-cpp'
-      uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
-      with:
-        path: ${{ github.workspace }}/.ccache
-        key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+          (cd build && make -j$(nproc))
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9

autotest/gcore/numpy_rw.py

@@ -983,3 +983,35 @@ def test_numpy_rw_band_read_as_array_getlasterrormsg():
     with gdal.quiet_errors():
         assert ds.GetRasterBand(1).ReadAsArray() is None
     assert gdal.GetLastErrorMsg() != ""
+
+
+###############################################################################
+# Test a band read into a masked array
+
+
+def test_numpy_rw_masked_array_1():
+
+    ds = gdal.Open("data/byte.tif")
+
+    band = ds.GetRasterBand(1)
+
+    masked_arr = band.ReadAsMaskedArray()
+
+    assert not numpy.any(masked_arr.mask)
+
+
+def test_numpy_rw_masked_array_2():
+
+    ds = gdal.Open("data/test3_with_mask_8bit.tif")
+
+    band = ds.GetRasterBand(1)
+
+    arr = band.ReadAsArray()
+    mask = band.GetMaskBand().ReadAsArray()
+
+    masked_arr = band.ReadAsMaskedArray()
+
+    assert not numpy.any(masked_arr.mask[mask == 255])
+    assert numpy.all(masked_arr.mask[mask != 255])
+
+    assert masked_arr.sum() == arr[mask == 255].sum()

autotest/gcore/tiff_write.py

@@ -414,20 +414,8 @@ def test_tiff_write_13():
     cs = ds.GetRasterBand(3).Checksum()
     ds = None
 
-    size = os.stat("tmp/sasha.tif").st_size
-
     gdaltest.tiff_drv.Delete("tmp/sasha.tif")
-    assert cs in (
-        17347,
-        14445,
-        14135,  # libjpeg 9e
-    )
-
-    md = gdaltest.tiff_drv.GetMetadata()
-    if md["LIBTIFF"] == "INTERNAL":
-        # 22816 with libjpeg-6b or libjpeg-turbo
-        # 22828 with libjpeg-9d
-        assert size <= 22828, "fail: bad size"
+    assert cs in (16612,)
 
 
 ###############################################################################
@@ -9415,29 +9403,144 @@ def test_tiff_write_lerc_float(gdalDataType, structType):
 
 
 ###############################################################################
-# Test LERC compression withFloat32/Float64 and nan
+
+
+def lerc_version_at_least_3():
+
+    LERC_VERSION_MAJOR = gdal.GetDriverByName("GTiff").GetMetadataItem(
+        "LERC_VERSION_MAJOR", "LERC"
+    )
+    return LERC_VERSION_MAJOR and int(LERC_VERSION_MAJOR) >= 3
+
+
+###############################################################################
+# Test LERC compression with Float32/Float64 and nan
 
 
 @pytest.mark.parametrize(
-    "gdalDataType,structType", [[gdal.GDT_Float32, "f"], [gdal.GDT_Float64, "d"]]
+    "gdalDataType,structType",
+    [
+        (gdal.GDT_Float32, "f"),
+        (gdal.GDT_Float64, "d"),
+    ],
+)
+@pytest.mark.parametrize("repeat", [1, 100])
+@pytest.mark.parametrize("interleave", ["PIXEL", "BAND"])
+@pytest.mark.parametrize(
+    "values",
+    [
+        [(0.5,)],
+        [(0.5, 1.5)],
+        [(float("nan"),)],
+        [(0.5, float("nan"))],
+        [(0.5, 0.75, 1), (1.5, 1.75, 2)],
+        [(float("nan"),), (float("nan"),)],
+        [(0.5, float("nan"), 1), (1.5, float("nan"), 2)],
+        [
+            (0.5, float("nan"), 1),
+            (1.5, 1.75, float("nan")),
+        ],  # This one requires liblerc >= 3.0 since we need multiple masks
+    ],
 )
 @pytest.mark.require_creation_option("GTiff", "LERC")
-def test_tiff_write_lerc_float_with_nan(gdalDataType, structType):
+def test_tiff_write_lerc_float_with_nan(
+    gdalDataType, structType, values, repeat, interleave
+):
 
-    src_ds = gdal.GetDriverByName("MEM").Create("", 2, 1, 1, gdalDataType)
-    src_ds.GetRasterBand(1).WriteRaster(
-        0, 0, 2, 1, struct.pack(structType * 2, 0.5, float("nan"))
-    )
+    bandCount = len(values)
+
+    if (
+        bandCount == 2
+        and True in [math.isnan(x) for x in values[0]]
+        and not (
+            check_libtiff_internal_or_at_least(4, 6, 1) and lerc_version_at_least_3()
+        )
+    ):
+        pytest.skip(
+            "multiple band with NaN in same strile only supported if libtiff >= 4.6.1 and liblerc >= 3.0"
+        )
+
+    width = len(values[0] * repeat)
+    src_ds = gdal.GetDriverByName("MEM").Create("", width, 1, bandCount, gdalDataType)
+    for i in range(bandCount):
+        src_ds.GetRasterBand(i + 1).WriteRaster(
+            0, 0, width, 1, array.array(structType, values[i] * repeat).tobytes()
+        )
     filename = "/vsimem/test.tif"
-    gdaltest.tiff_drv.CreateCopy(filename, src_ds, options=["COMPRESS=LERC"])
+    gdaltest.tiff_drv.CreateCopy(
+        filename, src_ds, options=["COMPRESS=LERC", "INTERLEAVE=" + interleave]
+    )
     ds = gdal.Open(filename)
-    got_data = struct.unpack(structType * 2, ds.ReadRaster())
-    assert got_data[0] == 0.5
-    assert math.isnan(got_data[1])
+    for i in range(bandCount):
+        got_data = struct.unpack(
+            structType * width, ds.GetRasterBand(i + 1).ReadRaster()
+        )
+        for j in range(width):
+            if math.isnan((values[i] * repeat)[j]):
+                assert math.isnan(got_data[j])
+            else:
+                assert got_data[j] == (values[i] * repeat)[j]
     ds = None
     gdal.Unlink(filename)
 
 
+###############################################################################
+
+
+@pytest.mark.parametrize("tiled", [False, True])
+@pytest.mark.require_creation_option("GTiff", "LERC")
+def test_tiff_write_lerc_float_with_nan_random(tmp_vsimem, tiled):
+
+    """Stress test the floating-point LERC encoder, with several masks per strile"""
+
+    width = 128
+    height = 128
+    bands = 100
+    src_ds = gdal.GetDriverByName("MEM").Create(
+        "", width, height, bands, gdal.GDT_Float32
+    )
+
+    import random
+
+    band_values = []
+    for i in range(bands):
+        # Generate random float values, but with at least 1/3 of nan in them in
+        # some bands and 2/3 in others
+        values = [int(random.random() * ((1 << 32) - 1)) for _ in range(width * height)]
+        values = array.array("I", values).tobytes()
+        values = [
+            x if random.random() > (0.33 if (i % 2) == 0 else 0.67) else float("nan")
+            for x in struct.unpack("f" * (width * height), values)
+        ]
+        band_values.append(values)
+        values = array.array("f", values).tobytes()
+        src_ds.GetRasterBand(i + 1).WriteRaster(0, 0, width, height, values)
+
+    filename = str(tmp_vsimem / "test_tiff_write_lerc_float_with_nan_random.tif")
+    if tiled:
+        options = ["COMPRESS=LERC", "TILED=YES", "BLOCKXSIZE=96", "BLOCKYSIZE=112"]
+    else:
+        options = ["COMPRESS=LERC", "BLOCKYSIZE=112"]
+    gdaltest.tiff_drv.CreateCopy(filename, src_ds, options=options)
+
+    if not (check_libtiff_internal_or_at_least(4, 6, 1) and lerc_version_at_least_3()):
+        pytest.skip(
+            "multiple band with NaN in same strile only supported if libtiff >= 4.6.1 and liblerc >= 3.0"
+        )
+
+    ds = gdal.Open(filename)
+    for i in range(bands):
+        got_data = struct.unpack(
+            "f" * (width * height), ds.GetRasterBand(i + 1).ReadRaster()
+        )
+        for j in range(width * height):
+            if math.isnan(band_values[i][j]):
+                assert math.isnan(got_data[j])
+            else:
+                assert got_data[j] == band_values[i][j]
+    ds = None
+
+
 ###############################################################################
 # Test JXL compression