[REVIEW] Upgrade arrow to 4.0.1

ci/gpu/build.sh

@@ -85,6 +85,7 @@ gpuci_conda_retry install -y \
 # https://docs.rapids.ai/maintainers/depmgmt/
 # gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env
 # gpuci_conda_retry install -y "your-pkg=1.0.0"
+gpuci_conda_retry install -y "arrow-cpp=4.0.1 pyarrow=4.0.1"
 
 
 gpuci_logger "Check compiler versions"

conda/environments/cudf_dev_cuda11.0.yml

@@ -17,7 +17,7 @@ dependencies:
   - numba>=0.53.1
   - numpy
   - pandas>=1.0,<1.3.0dev0
-  - pyarrow=1.0.1
+  - pyarrow=4.0.1
   - fastavro>=0.22.9
   - notebook>=0.5.0
   - cython>=0.29,<0.30
@@ -44,8 +44,8 @@ dependencies:
   - dask>=2021.4.0,<=2021.5.1
   - distributed>=2.22.0,<=2021.5.1
   - streamz
+  - arrow-cpp=4.0.1
   - dlpack>=0.5,<0.6.0a0
-  - arrow-cpp=1.0.1
   - arrow-cpp-proc * cuda
   - double-conversion
   - rapidjson

conda/environments/cudf_dev_cuda11.2.yml

@@ -17,7 +17,7 @@ dependencies:
   - numba>=0.53.1
   - numpy
   - pandas>=1.0,<1.3.0dev0
-  - pyarrow=1.0.1
+  - pyarrow=4.0.1
   - fastavro>=0.22.9
   - notebook>=0.5.0
   - cython>=0.29,<0.30
@@ -44,8 +44,8 @@ dependencies:
   - dask>=2021.4.0,<=2021.5.1
   - distributed>=2.22.0,<=2021.5.1
   - streamz
+  - arrow-cpp=4.0.1
   - dlpack>=0.5,<0.6.0a0
-  - arrow-cpp=1.0.1
   - arrow-cpp-proc * cuda
   - double-conversion
   - rapidjson

conda/recipes/cudf/meta.yaml

@@ -30,7 +30,7 @@ requirements:
     - setuptools
     - numba >=0.53.1
     - dlpack>=0.5,<0.6.0a0
-    - pyarrow 1.0.1
+    - pyarrow 4.0.1
     - libcudf {{ version }}
     - rmm {{ minor_version }}
     - cudatoolkit {{ cuda_version }}

conda/recipes/libcudf/meta.yaml

@@ -37,7 +37,7 @@ requirements:
   host:
     - librmm {{ minor_version }}.*
     - cudatoolkit {{ cuda_version }}.*
-    - arrow-cpp 1.0.1
+    - arrow-cpp 4.0.1
     - arrow-cpp-proc * cuda
     - dlpack>=0.5,<0.6.0a0
   run:

cpp/cmake/thirdparty/CUDF_GetArrow.cmake

@@ -121,6 +121,6 @@ function(find_and_configure_arrow VERSION BUILD_STATIC)
 
 endfunction()
 
-set(CUDF_VERSION_Arrow 1.0.1)
+set(CUDF_VERSION_Arrow 4.0.1)
 
 find_and_configure_arrow(${CUDF_VERSION_Arrow} ${CUDF_USE_ARROW_STATIC})

python/cudf/cudf/_fuzz_testing/avro.py

@@ -15,7 +15,7 @@
     pandas_to_avro,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",

python/cudf/cudf/_fuzz_testing/csv.py

@@ -12,7 +12,7 @@
     _generate_rand_meta,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 from cudf.utils.dtypes import pandas_dtypes_to_cudf_dtypes
 
 logging.basicConfig(

python/cudf/cudf/_fuzz_testing/json.py

@@ -13,7 +13,7 @@
     _generate_rand_meta,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 from cudf.utils.dtypes import pandas_dtypes_to_cudf_dtypes
 
 logging.basicConfig(

python/cudf/cudf/_fuzz_testing/orc.py

@@ -16,7 +16,7 @@
     pandas_to_orc,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",

python/cudf/cudf/_fuzz_testing/parquet.py

@@ -12,7 +12,7 @@
     _generate_rand_meta,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",

python/cudf/cudf/_fuzz_testing/tests/fuzz_test_csv.py

@@ -13,7 +13,7 @@
     compare_content,
     run_test,
 )
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 
 
 @pythonfuzz(data_handle=CSVReader)

python/cudf/cudf/_fuzz_testing/tests/fuzz_test_json.py

@@ -9,7 +9,7 @@
 from cudf._fuzz_testing.json import JSONReader, JSONWriter
 from cudf._fuzz_testing.main import pythonfuzz
 from cudf._fuzz_testing.utils import ALL_POSSIBLE_VALUES, run_test
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 
 
 @pythonfuzz(data_handle=JSONReader)

python/cudf/cudf/_fuzz_testing/utils.py

@@ -9,7 +9,7 @@
 import pyorc
 
 import cudf
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 from cudf.utils.dtypes import (
     pandas_dtypes_to_cudf_dtypes,
     pyarrow_dtypes_to_pandas_dtypes,

python/cudf/cudf/_lib/gpuarrow.pyx

@@ -15,15 +15,15 @@ from pyarrow.includes.libarrow cimport (
     CRecordBatchStreamReader
 )
 from pyarrow.lib cimport (
-    _CRecordBatchReader,
+    RecordBatchReader,
     Buffer,
     Schema,
     pyarrow_wrap_schema
 )
 import pyarrow as pa
 
 
-cdef class CudaRecordBatchStreamReader(_CRecordBatchReader):
+cdef class CudaRecordBatchStreamReader(RecordBatchReader):
     cdef:
         CIpcReadOptions options
 

python/cudf/cudf/_lib/utils.pyx

@@ -138,12 +138,16 @@ cpdef generate_pandas_metadata(Table table, index):
             index_descriptors.append(descr)
 
     metadata = pa.pandas_compat.construct_metadata(
-        table,
-        col_names,
-        index_levels,
-        index_descriptors,
-        index,
-        types,
+        columns_to_convert=[
+            col
+            for col in table._data.columns
+        ],
+        df=table,
+        column_names=col_names,
+        index_levels=index_levels,
+        index_descriptors=index_descriptors,
+        preserve_index=index,
+        types=types,
     )
 
     md_dict = json.loads(metadata[b"pandas"])

python/cudf/cudf/core/dataframe.py

@@ -5666,11 +5666,12 @@ def to_arrow(self, preserve_index=True):
 
         out = super(DataFrame, data).to_arrow()
         metadata = pa.pandas_compat.construct_metadata(
-            self,
-            out.schema.names,
-            [self.index],
-            index_descr,
-            preserve_index,
+            columns_to_convert=[self[col] for col in self._data.names],
+            df=self,
+            column_names=out.schema.names,
+            index_levels=[self.index],
+            index_descriptors=index_descr,
+            preserve_index=preserve_index,
             types=out.schema.types,
         )
 

python/cudf/cudf/tests/utils.py → python/cudf/cudf/testing/_utils.py

@@ -16,6 +16,8 @@
 from cudf.core.column.datetime import _numpy_to_pandas_conversion
 from cudf.utils import dtypes as dtypeutils
 
+_JOIN_TYPES = ("left", "inner", "outer", "right", "leftanti", "leftsemi")
+
 supported_numpy_dtypes = [
     "bool",
     "int8",
@@ -308,5 +310,36 @@ def does_not_raise():
     yield
 
 
+def assert_join_results_equal(expect, got, how, **kwargs):
+    if how not in _JOIN_TYPES:
+        raise ValueError(f"Unrecognized join type {how}")
+    if how == "right":
+        got = got[expect.columns]
+
+    if isinstance(expect, (pd.Series, cudf.Series)):
+        return assert_eq(
+            expect.sort_values().reset_index(drop=True),
+            got.sort_values().reset_index(drop=True),
+            **kwargs,
+        )
+    elif isinstance(expect, (pd.DataFrame, cudf.DataFrame)):
+        if not len(
+            expect.columns
+        ):  # can't sort_values() on a df without columns
+            return assert_eq(expect, got, **kwargs)
+
+        assert_eq(
+            expect.sort_values(expect.columns.to_list()).reset_index(
+                drop=True
+            ),
+            got.sort_values(got.columns.to_list()).reset_index(drop=True),
+            **kwargs,
+        )
+    elif isinstance(expect, (pd.Index, cudf.Index)):
+        return assert_eq(expect.sort_values(), got.sort_values(), **kwargs)
+    else:
+        raise ValueError(f"Not a join result: {type(expect).__name__}")
+
+
 def xfail_param(param, **kwargs):
     return pytest.param(param, marks=pytest.mark.xfail(**kwargs))

python/cudf/cudf/tests/test_apply_rows.py

@@ -2,7 +2,7 @@
 
 import cudf
 from cudf.core.column import column
-from cudf.tests.utils import assert_eq, gen_rand_series
+from cudf.testing._utils import assert_eq, gen_rand_series
 
 
 def _kernel_multiply(a, b, out):

python/cudf/cudf/tests/test_applymap.py

@@ -7,7 +7,7 @@
 import pytest
 
 from cudf import Series
-from cudf.tests import utils
+from cudf.testing import _utils as utils
 
 
 @pytest.mark.parametrize(

python/cudf/cudf/tests/test_array_function.py

@@ -4,7 +4,7 @@
 import pytest
 
 import cudf
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 from cudf.utils.utils import IS_NEP18_ACTIVE
 
 missing_arrfunc_cond = not IS_NEP18_ACTIVE

python/cudf/cudf/tests/test_array_ufunc.py

@@ -1,9 +1,10 @@
-import cudf
-import numpy as np
 import cupy as cp
+import numpy as np
 import pandas as pd
 import pytest
-from cudf.tests.utils import assert_eq
+
+import cudf
+from cudf.testing._utils import assert_eq
 
 
 @pytest.fixture

python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py

@@ -18,7 +18,7 @@
 import pytest
 
 import cudf
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 
 
 def cudf_from_avro_util(schema, records):

python/cudf/cudf/tests/test_binops.py

@@ -14,7 +14,7 @@
 import cudf
 from cudf.core import Series
 from cudf.core.index import as_index
-from cudf.tests import utils
+from cudf.testing import _utils as utils
 from cudf.utils.dtypes import (
     BOOL_TYPES,
     DATETIME_TYPES,
@@ -2242,7 +2242,7 @@ def test_binops_decimal(args):
         ),
     ],
 )
-@pytest.mark.parametrize("integer_dtype", cudf.tests.utils.INTEGER_TYPES)
+@pytest.mark.parametrize("integer_dtype", utils.INTEGER_TYPES)
 @pytest.mark.parametrize("reflected", [True, False])
 def test_binops_decimal_comp_mixed_integer(args, integer_dtype, reflected):
     """

python/cudf/cudf/tests/test_categorical.py

@@ -9,7 +9,7 @@
 
 import cudf
 from cudf.core._compat import PANDAS_GE_110
-from cudf.tests.utils import assert_eq, assert_exceptions_equal
+from cudf.testing._utils import assert_eq, assert_exceptions_equal
 
 
 @pytest.fixture

python/cudf/cudf/tests/test_column.py

@@ -8,7 +8,7 @@
 import cudf
 from cudf._lib.transform import mask_to_bools
 from cudf.core.column.column import as_column
-from cudf.tests.utils import assert_eq, assert_exceptions_equal
+from cudf.testing._utils import assert_eq, assert_exceptions_equal
 from cudf.utils import dtypes as dtypeutils
 
 dtypes = sorted(
@@ -140,8 +140,8 @@ def test_column_series_multi_dim(data):
 @pytest.mark.parametrize(
     ("data", "error"),
     [
-        ([1, "1.0", "2", -3], TypeError),
-        ([np.nan, 0, "null", cp.nan], TypeError),
+        ([1, "1.0", "2", -3], pa.lib.ArrowInvalid),
+        ([np.nan, 0, "null", cp.nan], pa.lib.ArrowInvalid),
         (
             [np.int32(4), np.float64(1.5), np.float32(1.290994), np.int8(0)],
             None,
@@ -152,7 +152,7 @@ def test_column_mixed_dtype(data, error):
     if error is None:
         cudf.Series(data)
     else:
-        with pytest.raises(TypeError):
+        with pytest.raises(error):
             cudf.Series(data)
 
 

python/cudf/cudf/tests/test_column_accessor.py

@@ -6,7 +6,7 @@
 
 import cudf
 from cudf.core.column_accessor import ColumnAccessor
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 
 simple_test_data = [
     {},

python/cudf/cudf/tests/test_concat.py

@@ -1,16 +1,16 @@
 # Copyright (c) 2018-2021, NVIDIA CORPORATION.
 
 import re
+from decimal import Decimal
 
 import numpy as np
 import pandas as pd
 import pytest
-from decimal import Decimal
 
 import cudf as gd
-from cudf.tests.utils import assert_eq, assert_exceptions_equal
-from cudf.utils.dtypes import is_categorical_dtype
 from cudf.core.dtypes import Decimal64Dtype
+from cudf.testing._utils import assert_eq, assert_exceptions_equal
+from cudf.utils.dtypes import is_categorical_dtype
 
 
 def make_frames(index=None, nulls="none"):

python/cudf/cudf/tests/test_contains.py

@@ -6,7 +6,7 @@
 
 from cudf import Series
 from cudf.core.index import RangeIndex, as_index
-from cudf.tests.utils import (
+from cudf.testing._utils import (
     DATETIME_TYPES,
     NUMERIC_TYPES,
     TIMEDELTA_TYPES,