From e600e47f3efd8b6f20e5bfeb265c934f8f8ba8df Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 29 Mar 2022 16:11:05 -0700 Subject: [PATCH 1/7] refactor memory_usage --- python/cudf/cudf/core/dataframe.py | 6 ++++-- python/cudf/cudf/core/frame.py | 4 +++- python/cudf/cudf/core/index.py | 2 +- python/cudf/cudf/core/indexed_frame.py | 7 ++++--- python/cudf/cudf/core/multiindex.py | 2 +- python/cudf/cudf/core/series.py | 2 +- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 17cac3593a3..379048fe73d 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1339,8 +1339,10 @@ def _slice(self: T, arg: slice) -> T: @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): - return Series( - {str(k): v for k, v in super().memory_usage(index, deep).items()} + col_names, mem_usage = super().memory_usage(index, deep) + return Series._from_data( + data={None: as_column(mem_usage)}, + index=as_index([str(name) for name in col_names]), ) @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index a84606b0953..2afd500d093 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -344,7 +344,9 @@ def memory_usage(self, deep=False): "The deep parameter is ignored and is only included " "for pandas compatibility." ) - return {name: col.memory_usage for name, col in self._data.items()} + return list(self._data.names), [ + col.memory_usage for col in self._data.columns + ] def __len__(self): return self._num_rows diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 7df5be3f692..c25bc3d6321 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -914,7 +914,7 @@ def _concat(cls, objs): @_cudf_nvtx_annotate def memory_usage(self, deep=False): - return sum(super().memory_usage(deep=deep).values()) + return sum(super().memory_usage(deep=deep)[1]) @_cudf_nvtx_annotate def equals(self, other, **kwargs): diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index c5c2322d95a..fab99010698 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -704,10 +704,11 @@ def memory_usage(self, index=True, deep=False): >>> s.memory_usage(index=False) 24 """ - usage = super().memory_usage(deep=deep) + col_names, usage = super().memory_usage(deep=deep) if index: - usage["Index"] = self.index.memory_usage() - return usage + col_names.append("Index") + usage.append(self.index.memory_usage()) + return col_names, usage def hash_values(self, method="murmur3"): """Compute the hash of values in this column. diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 39228f034d4..9eafb4528e4 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1474,7 +1474,7 @@ def _clean_nulls_from_index(self): @_cudf_nvtx_annotate def memory_usage(self, deep=False): - usage = sum(super().memory_usage(deep=deep).values()) + usage = sum(super().memory_usage(deep=deep)[1]) if self.levels: for level in self.levels: usage += level.memory_usage(deep=deep) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 0ea02edb924..f72587cfed1 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -856,7 +856,7 @@ def to_frame(self, name=None): @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): - return sum(super().memory_usage(index, deep).values()) + return sum(super().memory_usage(index, deep)[1]) @_cudf_nvtx_annotate def __array_function__(self, func, types, args, kwargs): From f232a74cf6bdb0a94c154f491f6551c706b51501 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 30 Mar 2022 10:18:36 -0700 Subject: [PATCH 2/7] address reviews --- python/cudf/cudf/core/dataframe.py | 13 ++++++++++--- python/cudf/cudf/core/frame.py | 9 +-------- python/cudf/cudf/core/index.py | 2 +- python/cudf/cudf/core/indexed_frame.py | 6 +----- python/cudf/cudf/core/multiindex.py | 2 +- python/cudf/cudf/core/series.py | 4 +++- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 379048fe73d..3e0d4a86451 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1339,10 +1339,17 @@ def _slice(self: T, arg: slice) -> T: @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): - col_names, mem_usage = super().memory_usage(index, deep) return Series._from_data( - data={None: as_column(mem_usage)}, - index=as_index([str(name) for name in col_names]), + data={ + None: as_column( + [col.memory_usage for col in self._data.columns] + + ([self.index.memory_usage()] if index else []) + ) + }, + index=as_index( + [str(name) for name in self._data.names] + + (["Index"] if index else []) + ), ) @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 2afd500d093..75c6e4d0964 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -339,14 +339,7 @@ def memory_usage(self, deep=False): ------- The total bytes used. """ - if deep: - warnings.warn( - "The deep parameter is ignored and is only included " - "for pandas compatibility." - ) - return list(self._data.names), [ - col.memory_usage for col in self._data.columns - ] + raise NotImplementedError def __len__(self): return self._num_rows diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index c25bc3d6321..a31fe4c3b99 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -914,7 +914,7 @@ def _concat(cls, objs): @_cudf_nvtx_annotate def memory_usage(self, deep=False): - return sum(super().memory_usage(deep=deep)[1]) + return self._column.memory_usage @_cudf_nvtx_annotate def equals(self, other, **kwargs): diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index fab99010698..458fc16c511 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -704,11 +704,7 @@ def memory_usage(self, index=True, deep=False): >>> s.memory_usage(index=False) 24 """ - col_names, usage = super().memory_usage(deep=deep) - if index: - col_names.append("Index") - usage.append(self.index.memory_usage()) - return col_names, usage + raise NotImplementedError def hash_values(self, method="murmur3"): """Compute the hash of values in this column. diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 9eafb4528e4..df4a781f150 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1474,7 +1474,7 @@ def _clean_nulls_from_index(self): @_cudf_nvtx_annotate def memory_usage(self, deep=False): - usage = sum(super().memory_usage(deep=deep)[1]) + usage = sum([col.memory_usage for col in self._data.columns]) if self.levels: for level in self.levels: usage += level.memory_usage(deep=deep) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index f72587cfed1..17862eb7bba 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -856,7 +856,9 @@ def to_frame(self, name=None): @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): - return sum(super().memory_usage(index, deep)[1]) + return self._column.memory_usage + ( + self.index.memory_usage() if index else 0 + ) @_cudf_nvtx_annotate def __array_function__(self, func, types, args, kwargs): From 816fd54321f860852e279714ece11945f1abbacf Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 30 Mar 2022 13:38:33 -0500 Subject: [PATCH 3/7] Update python/cudf/cudf/core/multiindex.py Co-authored-by: Bradley Dice --- python/cudf/cudf/core/multiindex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index df4a781f150..d80fb00942b 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1474,7 +1474,7 @@ def _clean_nulls_from_index(self): @_cudf_nvtx_annotate def memory_usage(self, deep=False): - usage = sum([col.memory_usage for col in self._data.columns]) + usage = sum(col.memory_usage for col in self._data.columns) if self.levels: for level in self.levels: usage += level.memory_usage(deep=deep) From 15c85284f632e33b4d1ed1e48ef5d5c79d1cf4d1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 30 Mar 2022 14:13:13 -0500 Subject: [PATCH 4/7] Update python/cudf/cudf/core/series.py Co-authored-by: Vyas Ramasubramani --- python/cudf/cudf/core/series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 17862eb7bba..8748b9775be 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -857,7 +857,7 @@ def to_frame(self, name=None): @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): return self._column.memory_usage + ( - self.index.memory_usage() if index else 0 + self._index.memory_usage() if index else 0 ) @_cudf_nvtx_annotate From 84951cbdf108cfc5d54a6c9222367c841ca314bd Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 30 Mar 2022 14:13:42 -0500 Subject: [PATCH 5/7] Update python/cudf/cudf/core/dataframe.py Co-authored-by: Vyas Ramasubramani --- python/cudf/cudf/core/dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 3e0d4a86451..6c720055e42 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1343,7 +1343,7 @@ def memory_usage(self, index=True, deep=False): data={ None: as_column( [col.memory_usage for col in self._data.columns] - + ([self.index.memory_usage()] if index else []) + + ([self._index.memory_usage()] if index else []) ) }, index=as_index( From 9df42a539b71ff56a684089bfcbd0bc834ab94af Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 30 Mar 2022 12:38:28 -0700 Subject: [PATCH 6/7] micro-optimize --- python/cudf/cudf/core/dataframe.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 6c720055e42..08a30729e7c 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1339,17 +1339,14 @@ def _slice(self: T, arg: slice) -> T: @_cudf_nvtx_annotate def memory_usage(self, index=True, deep=False): + mem_usage = [col.memory_usage for col in self._data.columns] + names = [str(name) for name in self._data.names] + if index: + mem_usage.append(self._index.memory_usage()) + names.append("Index") return Series._from_data( - data={ - None: as_column( - [col.memory_usage for col in self._data.columns] - + ([self._index.memory_usage()] if index else []) - ) - }, - index=as_index( - [str(name) for name in self._data.names] - + (["Index"] if index else []) - ), + data={None: as_column(mem_usage)}, + index=as_index(names), ) @_cudf_nvtx_annotate From 87589b1ec93a0c4d3975811bda25850f267d4287 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 30 Mar 2022 14:06:09 -0700 Subject: [PATCH 7/7] optimize sizeof --- python/dask_cudf/dask_cudf/backends.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index d1edfb071a2..36e3416c8a3 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -398,7 +398,10 @@ def group_split_cudf(df, c, k, ignore_index=False): @sizeof_dispatch.register(cudf.DataFrame) @_dask_cudf_nvtx_annotate def sizeof_cudf_dataframe(df): - return int(df.memory_usage().sum()) + return int( + sum(col.memory_usage for col in df._data.columns) + + df._index.memory_usage() + ) @sizeof_dispatch.register((cudf.Series, cudf.BaseIndex))