Skip to content

Commit

Permalink
deprecate open_file_options
Browse files Browse the repository at this point in the history
  • Loading branch information
lithomas1 committed Jul 9, 2024
1 parent 6311e52 commit 6fa66f0
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 6 deletions.
13 changes: 7 additions & 6 deletions python/cudf/cudf/tests/test_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,12 +379,13 @@ def test_read_parquet_filters(s3_base, s3so, pdf_ext, precache):
buffer.seek(0)
filters = [("String", "==", "Omega")]
with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}):
got = cudf.read_parquet(
f"s3://{bucket}/{fname}",
storage_options=s3so,
filters=filters,
open_file_options={"precache_options": {"method": precache}},
)
with pytest.warns(FutureWarning):
got = cudf.read_parquet(
f"s3://{bucket}/{fname}",
storage_options=s3so,
filters=filters,
open_file_options={"precache_options": {"method": precache}},
)

# All row-groups should be filtered out
assert_eq(pdf_ext.iloc[:0], got.reset_index(drop=True))
Expand Down
17 changes: 17 additions & 0 deletions python/cudf/cudf/utils/ioutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,11 @@
deactivate optimized precaching, set the "method" to `None` under the
"precache_options" key. Note that the `open_file_func` key can also be
used to specify a custom file-open function.
.. deprecated:: 24.08
`open_file_options` is deprecated as it was intended for
pyarrow file inputs, which will no longer be accepted as
input/output cudf readers/writers in the future.
bytes_per_thread : int, default None
Determines the number of bytes to be allocated per thread to read the
files in parallel. When there is a file of large size, we get slightly
Expand Down Expand Up @@ -1444,6 +1449,11 @@
open_file_options : dict, optional
Optional dictionary of keyword arguments to pass to
`_open_remote_files` (used for remote storage only).
.. deprecated:: 24.08
`open_file_options` is deprecated as it was intended for
pyarrow file inputs, which will no longer be accepted as
input/output cudf readers/writers in the future.
allow_raw_text_input : boolean, default False
If True, this indicates the input `path_or_data` could be a raw text
input and will not check for its existence in the filesystem. If False,
Expand Down Expand Up @@ -1745,6 +1755,13 @@ def get_reader_filepath_or_buffer(
# is deprecated
use_python_file_object = True

if open_file_options is not None:
warnings.warn(
"The 'open_file_options' keyword is deprecated and "
"will be removed in a future version.",
FutureWarning,
)

if isinstance(path_or_data, str):
# Get a filesystem object if one isn't already available
paths = [path_or_data]
Expand Down

0 comments on commit 6fa66f0

Please sign in to comment.