Refactor memory_usage to improve performance (#10537)

Refactored `Frame.memory_usage` to return a tuple of lists: (column_names, memory_usages).
Motivation for this change is to remove redundent steps i.e., `dict`(`Frame.memory_usage`)->`unpack & dict` (`DataFrame.memory_usage`)->`unpack dict`(in `Series.init`). Choosing to remove `dict` being returned by `Frame.memory_usage` will now result in a 10% faster execution of external API. Not a huge speedup but it quickly adds up when `dask_cudf` is used. 

```python
In [1]: import cudf

In [2]: df = cudf.DataFrame({'a':[1, 2, 3], 'b':['a', 'b', 'c'], 'd':[111, 123, 123]})

# THIS PR
In [3]: %timeit df.memory_usage()
198 µs ± 3.44 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)


# branch-22.06
In [3]: %timeit df.memory_usage()
219 µs ± 726 ns per loop (mean ± std. dev. of 7 runs, 1,000 loops each)


# branch-22.06
In [4]: %timeit dask_cudf.backends.sizeof_cudf_dataframe(df)
377 µs ± 5.67 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)

# this PR
In [6]: %timeit dask_cudf.backends.sizeof_cudf_dataframe(df)
1.8 µs ± 14 ns per loop (mean ± std. dev. of 7 runs, 1,000,000 loops each)

```

Authors:
  - GALI PREM SAGAR (https://github.com/galipremsagar)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Vyas Ramasubramani (https://github.com/vyasr)
  - Ashwin Srinath (https://github.com/shwina)

URL: #10537

python/cudf/cudf/core/dataframe.py

@@ -1339,8 +1339,14 @@ def _slice(self: T, arg: slice) -> T:
 
     @_cudf_nvtx_annotate
     def memory_usage(self, index=True, deep=False):
-        return Series(
-            {str(k): v for k, v in super().memory_usage(index, deep).items()}
+        mem_usage = [col.memory_usage for col in self._data.columns]
+        names = [str(name) for name in self._data.names]
+        if index:
+            mem_usage.append(self._index.memory_usage())
+            names.append("Index")
+        return Series._from_data(
+            data={None: as_column(mem_usage)},
+            index=as_index(names),
         )
 
     @_cudf_nvtx_annotate

python/cudf/cudf/core/frame.py

@@ -339,12 +339,7 @@ def memory_usage(self, deep=False):
         -------
         The total bytes used.
         """
-        if deep:
-            warnings.warn(
-                "The deep parameter is ignored and is only included "
-                "for pandas compatibility."
-            )
-        return {name: col.memory_usage for name, col in self._data.items()}
+        raise NotImplementedError
 
     def __len__(self):
         return self._num_rows

python/cudf/cudf/core/index.py

@@ -914,7 +914,7 @@ def _concat(cls, objs):
 
     @_cudf_nvtx_annotate
     def memory_usage(self, deep=False):
-        return sum(super().memory_usage(deep=deep).values())
+        return self._column.memory_usage
 
     @_cudf_nvtx_annotate
     def equals(self, other, **kwargs):

python/cudf/cudf/core/indexed_frame.py

@@ -704,10 +704,7 @@ def memory_usage(self, index=True, deep=False):
         >>> s.memory_usage(index=False)
         24
         """
-        usage = super().memory_usage(deep=deep)
-        if index:
-            usage["Index"] = self.index.memory_usage()
-        return usage
+        raise NotImplementedError
 
     def hash_values(self, method="murmur3"):
         """Compute the hash of values in this column.

python/cudf/cudf/core/multiindex.py

@@ -1474,7 +1474,7 @@ def _clean_nulls_from_index(self):
 
     @_cudf_nvtx_annotate
     def memory_usage(self, deep=False):
-        usage = sum(super().memory_usage(deep=deep).values())
+        usage = sum(col.memory_usage for col in self._data.columns)
         if self.levels:
             for level in self.levels:
                 usage += level.memory_usage(deep=deep)

python/cudf/cudf/core/series.py

@@ -856,7 +856,9 @@ def to_frame(self, name=None):
 
     @_cudf_nvtx_annotate
     def memory_usage(self, index=True, deep=False):
-        return sum(super().memory_usage(index, deep).values())
+        return self._column.memory_usage + (
+            self._index.memory_usage() if index else 0
+        )
 
     @_cudf_nvtx_annotate
     def __array_function__(self, func, types, args, kwargs):

python/dask_cudf/dask_cudf/backends.py

@@ -398,7 +398,10 @@ def group_split_cudf(df, c, k, ignore_index=False):
 @sizeof_dispatch.register(cudf.DataFrame)
 @_dask_cudf_nvtx_annotate
 def sizeof_cudf_dataframe(df):
-    return int(df.memory_usage().sum())
+    return int(
+        sum(col.memory_usage for col in df._data.columns)
+        + df._index.memory_usage()
+    )
 
 
 @sizeof_dispatch.register((cudf.Series, cudf.BaseIndex))