[REVIEW] DataFrame insert and creation optimizations

python/cudf/cudf/core/dataframe.py

@@ -831,8 +831,12 @@ def _init_from_dict_like(
                 self._data.multiindex = self._data.multiindex and isinstance(
                     col_name, tuple
                 )
-                self.insert(
-                    i, col_name, data[col_name], nan_as_null=nan_as_null
+                self._insert(
+                    i,
+                    col_name,
+                    data[col_name],
+                    nan_as_null=nan_as_null,
+                    ignore_index=True,
                 )
 
         if columns is not None:
@@ -1093,7 +1097,7 @@ def __setitem__(self, arg, value):
                 )
             else:
                 for col_name in self._data:
-                    scatter_map = arg[col_name]
+                    scatter_map = arg._data[col_name]
                     if is_scalar(value):
                         self._data[col_name][scatter_map] = value
                     else:
@@ -2551,6 +2555,16 @@ def insert(self, loc, name, value, nan_as_null=None):
             name or label of column to be inserted
         value : Series or array-like
         """
+        return self._insert(
+            loc=loc,
+            name=name,
+            value=value,
+            nan_as_null=nan_as_null,
+            ignore_index=False,
+        )
+
+    @annotate("DATAFRAME__INSERT", color="green", domain="cudf_python")
+    def _insert(self, loc, name, value, nan_as_null=None, ignore_index=False):
         if name in self._data:
             raise NameError(f"duplicated column name {name}")
 
@@ -2570,7 +2584,8 @@ def insert(self, loc, name, value, nan_as_null=None):
 
         if len(self) == 0:
             if isinstance(value, (pd.Series, Series)):
-                self._index = as_index(value.index)
+                if not ignore_index:
+                    self._index = as_index(value.index)
             elif len(value) > 0:
                 self._index = RangeIndex(start=0, stop=len(value))
                 new_data = self._data.__class__()
@@ -2583,9 +2598,11 @@ def insert(self, loc, name, value, nan_as_null=None):
                         )
                 self._data = new_data
         elif isinstance(value, (pd.Series, Series)):
-            value = Series(value, nan_as_null=nan_as_null)._align_to_index(
-                self._index, how="right", sort=False
-            )
+            value = Series(value, nan_as_null=nan_as_null)
+            if not ignore_index:
+                value = value._align_to_index(
+                    self._index, how="right", sort=False
+                )
 
         value = column.as_column(value, nan_as_null=nan_as_null)
 
@@ -4710,8 +4727,11 @@ def to_arrow(self, preserve_index=True):
                 for gen_name, col_name in zip(
                     gen_names, self.index._data.names
                 ):
-                    data.insert(
-                        data.shape[1], gen_name, self.index._data[col_name]
+                    data._insert(
+                        data.shape[1],
+                        gen_name,
+                        self.index._data[col_name],
+                        ignore_index=True,
                     )
                 descr = gen_names[0]
             index_descr.append(descr)
@@ -5704,7 +5724,7 @@ def select_dtypes(self, include=None, exclude=None):
         for k, col in self._data.items():
             infered_type = cudf_dtype_from_pydata_dtype(col.dtype)
             if infered_type in inclusion:
-                df.insert(len(df._data), k, col)
+                df._insert(len(df._data), k, col, ignore_index=True)
 
         return df
 
@@ -6511,7 +6531,12 @@ def _setitem_with_dataframe(
                 raise ValueError("Can not insert new column with a bool mask")
             else:
                 # handle append case
-                input_df.insert(len(input_df._data), col_1, replace_df[col_2])
+                input_df._insert(
+                    loc=len(input_df._data),
+                    name=col_1,
+                    value=replace_df[col_2],
+                    ignore_index=True,
+                )
 
 
 def extract_col(df, col):

python/cudf/cudf/core/frame.py

@@ -6806,10 +6806,12 @@ def _drop_rows_by_labels(
         join_res = working_df.join(to_join, how="leftanti")
 
         # 4. Reconstruct original layout, and rename
-        join_res.insert(
-            ilevel, name=join_res._index.name, value=join_res._index
+        join_res._insert(
+            ilevel,
+            name=join_res._index.name,
+            value=join_res._index,
+            ignore_index=True,
         )
-        join_res = join_res.reset_index(drop=True)
 
         midx = cudf.MultiIndex.from_frame(
             join_res.iloc[:, 0:idx_nlv], names=obj._index.names

python/cudf/cudf/core/groupby/groupby.py

@@ -286,10 +286,11 @@ def agg(self, func):
 
         if not self._as_index:
             for col_name in reversed(self.grouping._named_columns):
-                result.insert(
+                result._insert(
                     0,
                     col_name,
                     result.index.get_level_values(col_name)._values,
+                    ignore_index=True,
                 )
             result.index = cudf.core.index.RangeIndex(len(result))
 

python/cudf/cudf/utils/utils.py

@@ -67,15 +67,7 @@ def scalar_broadcast_to(scalar, size, dtype=None):
     scalar = to_cudf_compatible_scalar(scalar, dtype=dtype)
     dtype = scalar.dtype
 
-    if cudf.dtype(dtype).kind in ("O", "U"):
-        gather_map = column.full(size, 0, dtype="int32")
-        scalar_str_col = column.as_column([scalar], dtype="str")
-        return scalar_str_col[gather_map]
-    else:
-        out_col = column.column_empty(size, dtype=dtype)
-        if out_col.size != 0:
-            out_col.data_array_view[:] = scalar
-        return out_col
+    return cudf.core.column.full(size=size, fill_value=scalar, dtype=dtype)
 
 
 def initfunc(f):