From 13551916d874c42ad02a8b3090bc97e02b35a5fa Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 30 Mar 2022 19:46:49 -0500 Subject: [PATCH] Refactor `memory_usage` to improve performance (#10537) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactored `Frame.memory_usage` to return a tuple of lists: (column_names, memory_usages). Motivation for this change is to remove redundent steps i.e., `dict`(`Frame.memory_usage`)->`unpack & dict` (`DataFrame.memory_usage`)->`unpack dict`(in `Series.init`). Choosing to remove `dict` being returned by `Frame.memory_usage` will now result in a 10% faster execution of external API. Not a huge speedup but it quickly adds up when `dask_cudf` is used. ```python In [1]: import cudf In [2]: df = cudf.DataFrame({'a':[1, 2, 3], 'b':['a', 'b', 'c'], 'd':[111, 123, 123]}) # THIS PR In [3]: %timeit df.memory_usage() 198 µs ± 3.44 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each) # branch-22.06 In [3]: %timeit df.memory_usage() 219 µs ± 726 ns per loop (mean ± std. dev. of 7 runs, 1,000 loops each) # branch-22.06 In [4]: %timeit dask_cudf.backends.sizeof_cudf_dataframe(df) 377 µs ± 5.67 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each) # this PR In [6]: %timeit dask_cudf.backends.sizeof_cudf_dataframe(df) 1.8 µs ± 14 ns per loop (mean ± std. dev. of 7 runs, 1,000,000 loops each) ``` Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/10537 --- python/cudf/cudf/core/dataframe.py | 10 ++++++++-- python/cudf/cudf/core/frame.py | 7 +------ python/cudf/cudf/core/index.py | 2 +- python/cudf/cudf/core/indexed_frame.py | 5 +---- python/cudf/cudf/core/multiindex.py | 2 +- python/cudf/cudf/core/series.py | 4 +++- python/dask_cudf/dask_cudf/backends.py | 5 ++++- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 17cac3593a3..08a30729e7c 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1339,8 +1339,14 @@ def _slice(self: T, arg: slice) -> T: @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): - return Series( - {str(k): v for k, v in super().memory_usage(index, deep).items()} + mem_usage = [col.memory_usage for col in self._data.columns] + names = [str(name) for name in self._data.names] + if index: + mem_usage.append(self._index.memory_usage()) + names.append("Index") + return Series._from_data( + data={None: as_column(mem_usage)}, + index=as_index(names), ) @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index a84606b0953..75c6e4d0964 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -339,12 +339,7 @@ def memory_usage(self, deep=False): ------- The total bytes used. """ - if deep: - warnings.warn( - "The deep parameter is ignored and is only included " - "for pandas compatibility." - ) - return {name: col.memory_usage for name, col in self._data.items()} + raise NotImplementedError def __len__(self): return self._num_rows diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 7df5be3f692..a31fe4c3b99 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -914,7 +914,7 @@ def _concat(cls, objs): @_cudf_nvtx_annotate def memory_usage(self, deep=False): - return sum(super().memory_usage(deep=deep).values()) + return self._column.memory_usage @_cudf_nvtx_annotate def equals(self, other, **kwargs): diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index c5c2322d95a..458fc16c511 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -704,10 +704,7 @@ def memory_usage(self, index=True, deep=False): >>> s.memory_usage(index=False) 24 """ - usage = super().memory_usage(deep=deep) - if index: - usage["Index"] = self.index.memory_usage() - return usage + raise NotImplementedError def hash_values(self, method="murmur3"): """Compute the hash of values in this column. diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 39228f034d4..d80fb00942b 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1474,7 +1474,7 @@ def _clean_nulls_from_index(self): @_cudf_nvtx_annotate def memory_usage(self, deep=False): - usage = sum(super().memory_usage(deep=deep).values()) + usage = sum(col.memory_usage for col in self._data.columns) if self.levels: for level in self.levels: usage += level.memory_usage(deep=deep) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 0ea02edb924..8748b9775be 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -856,7 +856,9 @@ def to_frame(self, name=None): @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): - return sum(super().memory_usage(index, deep).values()) + return self._column.memory_usage + ( + self._index.memory_usage() if index else 0 + ) @_cudf_nvtx_annotate def __array_function__(self, func, types, args, kwargs): diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index d1edfb071a2..36e3416c8a3 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -398,7 +398,10 @@ def group_split_cudf(df, c, k, ignore_index=False): @sizeof_dispatch.register(cudf.DataFrame) @_dask_cudf_nvtx_annotate def sizeof_cudf_dataframe(df): - return int(df.memory_usage().sum()) + return int( + sum(col.memory_usage for col in df._data.columns) + + df._index.memory_usage() + ) @sizeof_dispatch.register((cudf.Series, cudf.BaseIndex))