IndexError in test_read_parquet_partitioned_filtered[True-files-pfilters1]

[jakirkham]

Seeing this test failure on CI:

=================================== FAILURES ===================================
_________ test_read_parquet_partitioned_filtered[True-files-pfilters1] _________
[gw1] linux -- Python 3.9.18 /opt/conda/envs/test/bin/python3.9

tmpdir = local('/tmp/pytest-of-root/pytest-0/popen-gw1/test_read_parquet_partitioned_3')
pfilters = [('b', '==', 'a'), ('c', '==', 1)], selection = 'files'
use_cat = True

    @pytest.mark.parametrize(
        "pfilters",
        [[("b", "==", "b")], [("b", "==", "a"), ("c", "==", 1)]],
    )
    @pytest.mark.parametrize("selection", ["directory", "files", "row-groups"])
    @pytest.mark.parametrize("use_cat", [True, False])
    def test_read_parquet_partitioned_filtered(
        tmpdir, pfilters, selection, use_cat
    ):
        path = str(tmpdir)
        size = 100
        df = cudf.DataFrame(
            {
                "a": np.arange(0, stop=size, dtype="int64"),
                "b": np.random.choice(list("abcd"), size=size),
                "c": np.random.choice(np.arange(4), size=size),
            }
        )
        df.to_parquet(path, partition_cols=["c", "b"])
    
        if selection == "files":
            # Pass in a list of paths
            fs = get_fs_token_paths(path)[0]
            read_path = fs.find(path)
            row_groups = None
        elif selection == "row-groups":
            # Pass in a list of paths AND row-group ids
            fs = get_fs_token_paths(path)[0]
            read_path = fs.find(path)
            row_groups = [[0] for p in read_path]
        else:
            # Pass in a directory path
            # (row-group selection not allowed in this case)
            read_path = path
            row_groups = None
    
        # Filter on partitioned columns
        expect = pd.read_parquet(read_path, filters=pfilters)
>       got = cudf.read_parquet(
            read_path,
            filters=pfilters,
            row_groups=row_groups,
            categorical_partitions=use_cat,
        )

tests/test_parquet.py:2144: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/conda/envs/test/lib/python3.9/site-packages/nvtx/nvtx.py:116: in inner
    result = func(*args, **kwargs)
/opt/conda/envs/test/lib/python3.9/site-packages/cudf/io/parquet.py:577: in read_parquet
    df = _parquet_to_frame(
/opt/conda/envs/test/lib/python3.9/site-packages/nvtx/nvtx.py:116: in inner
    result = func(*args, **kwargs)
/opt/conda/envs/test/lib/python3.9/site-packages/cudf/io/parquet.py:721: in _parquet_to_frame
    return _read_parquet(
/opt/conda/envs/test/lib/python3.9/site-packages/nvtx/nvtx.py:116: in inner
    result = func(*args, **kwargs)
/opt/conda/envs/test/lib/python3.9/site-packages/cudf/io/parquet.py:831: in _read_parquet
    return libparquet.read_parquet(
parquet.pyx:124: in cudf._lib.parquet.read_parquet
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   IndexError: list index out of range

parquet.pyx:275: IndexError
-------- generated xml file: /__w/cudf/cudf/test-results/junit-cudf.xml --------

Edit: Seen recently in an unrelated Doxygen build fix ( #15289 )

[jakirkham]

Looks like this was run into by the Spark team recently as well

#15219 (comment)

[vyasr]

Seems like this was fixed by #15296. Perhaps we were randomly generating invalid data on occasion. Feel free to reopen if we find a meaningful reproducer again.