[REVIEW] Upgrade arrow to 4.0.1

conda/environments/cudf_dev_cuda10.1.yml

@@ -18,7 +18,7 @@ dependencies:
   - numba>=0.49.0,!=0.51.0
   - numpy
   - pandas>=1.0,<1.3.0dev0
-  - pyarrow=1.0.1
+  - pyarrow=3.0.0
   - fastavro>=0.22.9
   - notebook>=0.5.0
   - cython>=0.29,<0.30
@@ -47,7 +47,7 @@ dependencies:
   - distributed>=2.22.0
   - streamz
   - dlpack
-  - arrow-cpp=1.0.1
+  - arrow-cpp=3.0.0
   - arrow-cpp-proc * cuda
   - boost-cpp>=1.72.0
   - double-conversion

conda/environments/cudf_dev_cuda10.2.yml

@@ -18,7 +18,7 @@ dependencies:
   - numba>=0.49,!=0.51.0
   - numpy
   - pandas>=1.0,<1.3.0dev0
-  - pyarrow=1.0.1
+  - pyarrow=3.0.0
   - fastavro>=0.22.9
   - notebook>=0.5.0
   - cython>=0.29,<0.30
@@ -47,7 +47,7 @@ dependencies:
   - distributed>=2.22.0
   - streamz
   - dlpack
-  - arrow-cpp=1.0.1
+  - arrow-cpp=3.0.0
   - arrow-cpp-proc * cuda
   - boost-cpp>=1.72.0
   - double-conversion

conda/environments/cudf_dev_cuda11.0.yml

@@ -18,7 +18,7 @@ dependencies:
   - numba>=0.49,!=0.51.0
   - numpy
   - pandas>=1.0,<1.3.0dev0
-  - pyarrow=1.0.1
+  - pyarrow=3.0.0
   - fastavro>=0.22.9
   - notebook>=0.5.0
   - cython>=0.29,<0.30
@@ -47,7 +47,7 @@ dependencies:
   - distributed>=2.22.0
   - streamz
   - dlpack
-  - arrow-cpp=1.0.1
+  - arrow-cpp=3.0.0
   - arrow-cpp-proc * cuda
   - boost-cpp>=1.72.0
   - double-conversion

conda/recipes/cudf/meta.yaml

@@ -27,7 +27,7 @@ requirements:
     - setuptools
     - numba >=0.49.0
     - dlpack
-    - pyarrow 1.0.1
+    - pyarrow 3.0.0
     - libcudf {{ version }}
     - rmm {{ minor_version }}
     - cudatoolkit {{ cuda_version }}

conda/recipes/libcudf/meta.yaml

@@ -37,7 +37,7 @@ requirements:
   host:
     - librmm {{ minor_version }}.*
     - cudatoolkit {{ cuda_version }}.*
-    - arrow-cpp 1.0.1
+    - arrow-cpp 3.0.0
     - arrow-cpp-proc * cuda
     - boost-cpp 1.72.0
     - dlpack

cpp/cmake/thirdparty/CUDF_GetArrow.cmake

@@ -118,6 +118,6 @@ function(find_and_configure_arrow VERSION BUILD_STATIC)
 
 endfunction()
 
-set(CUDF_VERSION_Arrow 1.0.1)
+set(CUDF_VERSION_Arrow 3.0.0)
 
 find_and_configure_arrow(${CUDF_VERSION_Arrow} ${ARROW_STATIC_LIB})

python/cudf/cudf/_fuzz_testing/avro.py

@@ -15,7 +15,7 @@
     pandas_to_avro,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",

python/cudf/cudf/_fuzz_testing/csv.py

@@ -12,7 +12,7 @@
     _generate_rand_meta,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 from cudf.utils.dtypes import pandas_dtypes_to_cudf_dtypes
 
 logging.basicConfig(

python/cudf/cudf/_fuzz_testing/json.py

@@ -13,7 +13,7 @@
     _generate_rand_meta,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 from cudf.utils.dtypes import pandas_dtypes_to_cudf_dtypes
 
 logging.basicConfig(

python/cudf/cudf/_fuzz_testing/orc.py

@@ -16,7 +16,7 @@
     pandas_to_orc,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",

python/cudf/cudf/_fuzz_testing/parquet.py

@@ -12,7 +12,7 @@
     _generate_rand_meta,
     pyarrow_to_pandas,
 )
-from cudf.tests import dataset_generator as dg
+from cudf.testing import dataset_generator as dg
 
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",

python/cudf/cudf/_fuzz_testing/tests/fuzz_test_csv.py

@@ -13,7 +13,7 @@
     compare_content,
     run_test,
 )
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 
 
 @pythonfuzz(data_handle=CSVReader)

python/cudf/cudf/_fuzz_testing/tests/fuzz_test_json.py

@@ -9,7 +9,7 @@
 from cudf._fuzz_testing.json import JSONReader, JSONWriter
 from cudf._fuzz_testing.main import pythonfuzz
 from cudf._fuzz_testing.utils import ALL_POSSIBLE_VALUES, run_test
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 
 
 @pythonfuzz(data_handle=JSONReader)

python/cudf/cudf/_fuzz_testing/utils.py

@@ -9,7 +9,7 @@
 import pyorc
 
 import cudf
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 from cudf.utils.dtypes import (
     pandas_dtypes_to_cudf_dtypes,
     pyarrow_dtypes_to_pandas_dtypes,

python/cudf/cudf/_lib/gpuarrow.pyx

@@ -15,15 +15,15 @@ from pyarrow.includes.libarrow cimport (
     CRecordBatchStreamReader
 )
 from pyarrow.lib cimport (
-    _CRecordBatchReader,
+    RecordBatchReader,
     Buffer,
     Schema,
     pyarrow_wrap_schema
 )
 import pyarrow as pa
 
 
-cdef class CudaRecordBatchStreamReader(_CRecordBatchReader):
+cdef class CudaRecordBatchStreamReader(RecordBatchReader):
     cdef:
         CIpcReadOptions options
 

python/cudf/cudf/_lib/utils.pyx

@@ -116,12 +116,19 @@ cpdef generate_pandas_metadata(Table table, index):
             index_descriptors.append(descr)
 
     metadata = pa.pandas_compat.construct_metadata(
-        table,
-        col_names,
-        index_levels,
-        index_descriptors,
-        index,
-        types,
+        columns_to_convert=[
+            col.to_pandas()
+            if isinstance(col, cudf.core.column.CategoricalColumn)
+            else
+            col
+            for col in table._data.columns
+        ],
+        df=table,
+        column_names=col_names,
+        index_levels=index_levels,
+        index_descriptors=index_descriptors,
+        preserve_index=index,
+        types=types,
     )
 
     md_dict = json.loads(metadata[b"pandas"])

python/cudf/cudf/comm/gpuarrow.py

@@ -1,4 +1,5 @@
-# Copyright (c) 2019-2020, NVIDIA CORPORATION.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+
 from collections import OrderedDict
 from collections.abc import Sequence
 
@@ -9,7 +10,7 @@
 from cudf._lib.gpuarrow import (
     CudaRecordBatchStreamReader as _CudaRecordBatchStreamReader,
 )
-from cudf.core import Series, column
+from cudf.core import DataFrame, Series, column
 from cudf.utils.utils import mask_bitsize, mask_dtype
 
 
@@ -33,19 +34,25 @@ def __init__(self, source, schema=None):
 
 class GpuArrowReader(Sequence):
     def __init__(self, schema, dev_ary):
-        self._table = CudaRecordBatchStreamReader(dev_ary, schema).read_all()
+        table = CudaRecordBatchStreamReader(dev_ary, schema).read_all()
+        self._df = DataFrame.from_arrow(table)
+        self._schema = pa.Schema.from_pandas(self._df)
 
     def __len__(self):
-        return self._table.num_columns
+        return len(self._df._data.names)
 
     def __getitem__(self, idx):
-        return GpuArrowNodeReader(self._table, idx)
+        return GpuArrowNodeReader(
+            schema=self._schema,
+            field=self._schema[idx],
+            series=self._df._data.columns[idx],
+        )
 
     def schema(self):
         """
         Return a pyarrow schema
         """
-        return self._table.schema
+        return self._schema
 
     def to_dict(self):
         """
@@ -58,18 +65,18 @@ def to_dict(self):
 
 
 class GpuArrowNodeReader(object):
-    def __init__(self, table, index):
-        self._table = table
-        self._field = table.schema[index]
-        self._series = Series(column.as_column(table.column(index)))
+    def __init__(self, schema, field, series):
+        self._schema = schema
+        self._field = field
+        self._series = Series(column.as_column(series))
         self._series.name = self.name
 
     def __len__(self):
         return len(self._series)
 
     @property
     def schema(self):
-        return self._table.schema
+        return self._schema
 
     @property
     def field_schema(self):

python/cudf/cudf/core/column/categorical.py

@@ -936,6 +936,7 @@ def ordered(self) -> Optional[bool]:
     def ordered(self, value: bool):
         self.dtype.ordered = value
 
+    # @property
     def cat(self, parent: ParentType = None):
         return CategoricalAccessor(self, parent=parent)
 

python/cudf/cudf/core/column/column.py

@@ -63,6 +63,7 @@
 from cudf.utils.utils import mask_dtype
 
 T = TypeVar("T", bound="ColumnBase")
+ParentType = Union["cudf.Series", "cudf.Index"]
 
 
 class ColumnBase(Column, Serializable):
@@ -188,6 +189,7 @@ def __sizeof__(self) -> int:
             n += bitmask_allocation_size_bytes(self.size)
         return n
 
+    # @property
     def cat(
         self, parent=None
     ) -> "cudf.core.column.categorical.CategoricalAccessor":

python/cudf/cudf/core/dataframe.py

@@ -5396,11 +5396,17 @@ def to_arrow(self, preserve_index=True):
 
         out = super(DataFrame, data).to_arrow()
         metadata = pa.pandas_compat.construct_metadata(
-            self,
-            out.schema.names,
-            [self.index],
-            index_descr,
-            preserve_index,
+            columns_to_convert=[
+                col.to_pandas()
+                if isinstance(col, cudf.core.column.CategoricalColumn)
+                else col
+                for col in self._data.columns
+            ],
+            df=self,
+            column_names=out.schema.names,
+            index_levels=[self.index],
+            index_descriptors=index_descr,
+            preserve_index=preserve_index,
             types=out.schema.types,
         )
 

python/cudf/cudf/tests/test_apply_rows.py

@@ -2,7 +2,7 @@
 
 import cudf
 from cudf.core.column import column
-from cudf.tests.utils import assert_eq, gen_rand_series
+from cudf.testing._utils import assert_eq, gen_rand_series
 
 
 def _kernel_multiply(a, b, out):

python/cudf/cudf/tests/test_applymap.py

@@ -7,7 +7,7 @@
 import pytest
 
 from cudf import Series
-from cudf.tests import utils
+from cudf.testing import utils
 
 
 @pytest.mark.parametrize(

python/cudf/cudf/tests/test_array_function.py

@@ -4,7 +4,7 @@
 import pytest
 
 import cudf
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 from cudf.utils.utils import IS_NEP18_ACTIVE
 
 missing_arrfunc_cond = not IS_NEP18_ACTIVE

python/cudf/cudf/tests/test_array_ufunc.py

@@ -1,9 +1,10 @@
-import cudf
-import numpy as np
 import cupy as cp
+import numpy as np
 import pandas as pd
 import pytest
-from cudf.tests.utils import assert_eq
+
+import cudf
+from cudf.testing._utils import assert_eq
 
 
 @pytest.fixture

python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py

@@ -18,7 +18,7 @@
 import pytest
 
 import cudf
-from cudf.tests.utils import assert_eq
+from cudf.testing._utils import assert_eq
 
 
 def cudf_from_avro_util(schema, records):

python/cudf/cudf/tests/test_binops.py

@@ -14,7 +14,7 @@
 import cudf
 from cudf.core import Series
 from cudf.core.index import as_index
-from cudf.tests import utils
+from cudf.testing import utils
 from cudf.utils.dtypes import (
     BOOL_TYPES,
     DATETIME_TYPES,

python/cudf/cudf/tests/test_categorical.py

@@ -9,7 +9,7 @@
 
 import cudf
 from cudf.core._compat import PANDAS_GE_110
-from cudf.tests.utils import assert_eq, assert_exceptions_equal
+from cudf.testing._utils import assert_eq, assert_exceptions_equal
 
 
 @pytest.fixture