From 4fef478f57fb27053f3d6f2ceab25ae635ec34a1 Mon Sep 17 00:00:00 2001 From: Haibin Lin Date: Wed, 16 Sep 2020 16:10:55 -0700 Subject: [PATCH] Fix a typo in comments (#415) * Update stage2.py Co-authored-by: Jeff Rasley --- deepspeed/runtime/zero/stage2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepspeed/runtime/zero/stage2.py b/deepspeed/runtime/zero/stage2.py index 3b88f26e0220..c8835d178eff 100755 --- a/deepspeed/runtime/zero/stage2.py +++ b/deepspeed/runtime/zero/stage2.py @@ -517,7 +517,7 @@ def independent_gradient_partition_epilogue(self): see_memory_usage(f"End ipg_epilogue") # resets all partition to no reduced - # sets remianing grads to the total number of grads in each partition + # sets remaining grads to the total number of grads in each partition # set is grad computed to false for all grads in partition def reset_partition_gradient_structures(self): total_partitions = dist.get_world_size(group=self.dp_process_group) @@ -617,7 +617,7 @@ def report_ipg_memory_usage(self, tag, param_elems): f"{tag}: elems in_bucket {self.elements_in_ipg_bucket} param {param_elems} max_percent {percent_of_bucket_size}" ) - ###############Idependent Partition Gradient ######################## + ############### Independent Partition Gradient ######################## def reduce_independent_p_g_buckets_and_remove_grads(self, param, i): if self.elements_in_ipg_bucket + param.numel() > self.reduce_bucket_size: self.report_ipg_memory_usage("In ipg_remove_grads before reduce_ipg_grads",