From 148ea74e19ac214c323e8f050f12c078eab0e84e Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Tue, 31 Jan 2023 10:39:30 -0800 Subject: [PATCH 1/4] Including support for Deepspeed 0.8.0 --- .../orttraining/python/training/optim/_ds_modifier.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/orttraining/orttraining/python/training/optim/_ds_modifier.py b/orttraining/orttraining/python/training/optim/_ds_modifier.py index 9e3470d33c8e1..645ecd152339a 100644 --- a/orttraining/orttraining/python/training/optim/_ds_modifier.py +++ b/orttraining/orttraining/python/training/optim/_ds_modifier.py @@ -39,7 +39,7 @@ def can_be_modified(self): # it's safe to update the version supporting list. Otherwise, or the file is moved or renamed, # we need to check the implementation of these functions in detail. ds_version = Version(deepspeed.__version__) - if ds_version > Version("0.7.3") or ds_version < Version("0.4.0"): + if ds_version > Version("0.8.0") or ds_version < Version("0.4.0"): warnings.warn( "Skip modifying optimizer because of unsupported DeepSpeed version {}, " "supported version: 0.4.0 - 0.7.3.".format(deepspeed.__version__), @@ -47,6 +47,14 @@ def can_be_modified(self): ) return False + try: + from deepspeed.accelerator import get_accelerator + except ImportError as e: + pass + else: + if not get_accelerator().device_name().startswith("cuda"): + return False + return self.check_requirements( ["has_overflow_serial", "get_grad_norm_direct", "has_overflow_partitioned_grads_serial"], require_apex=False, From 28fb80a95423254202e8fed838d39f9d616d40b7 Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Tue, 31 Jan 2023 17:50:57 -0800 Subject: [PATCH 2/4] adding warnings for more information --- .../orttraining/python/training/optim/_ds_modifier.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/orttraining/orttraining/python/training/optim/_ds_modifier.py b/orttraining/orttraining/python/training/optim/_ds_modifier.py index 645ecd152339a..9c8a6c45e8a21 100644 --- a/orttraining/orttraining/python/training/optim/_ds_modifier.py +++ b/orttraining/orttraining/python/training/optim/_ds_modifier.py @@ -50,9 +50,14 @@ def can_be_modified(self): try: from deepspeed.accelerator import get_accelerator except ImportError as e: + warnings.warn("Unable to import get_accelerator from deepspeed.accelerator", UserWarning) pass else: if not get_accelerator().device_name().startswith("cuda"): + warnings.warn( + "Skip modifying optimizer as device is not supported, " + "device name: {}".format(get_accelerator().device_name()), UserWarning, + ) return False return self.check_requirements( From c158b5786440e2baa31c4818a1d4911670092bd6 Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Tue, 31 Jan 2023 18:10:57 -0800 Subject: [PATCH 3/4] removing unnecessary pass statement --- orttraining/orttraining/python/training/optim/_ds_modifier.py | 1 - 1 file changed, 1 deletion(-) diff --git a/orttraining/orttraining/python/training/optim/_ds_modifier.py b/orttraining/orttraining/python/training/optim/_ds_modifier.py index 9c8a6c45e8a21..e593003c71b59 100644 --- a/orttraining/orttraining/python/training/optim/_ds_modifier.py +++ b/orttraining/orttraining/python/training/optim/_ds_modifier.py @@ -51,7 +51,6 @@ def can_be_modified(self): from deepspeed.accelerator import get_accelerator except ImportError as e: warnings.warn("Unable to import get_accelerator from deepspeed.accelerator", UserWarning) - pass else: if not get_accelerator().device_name().startswith("cuda"): warnings.warn( From 12c89442ebb48287267efbcaef262adaebda1e69 Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Tue, 31 Jan 2023 18:32:34 -0800 Subject: [PATCH 4/4] python format change --- orttraining/orttraining/python/training/optim/_ds_modifier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orttraining/orttraining/python/training/optim/_ds_modifier.py b/orttraining/orttraining/python/training/optim/_ds_modifier.py index e593003c71b59..dc2d4c091c65c 100644 --- a/orttraining/orttraining/python/training/optim/_ds_modifier.py +++ b/orttraining/orttraining/python/training/optim/_ds_modifier.py @@ -55,7 +55,8 @@ def can_be_modified(self): if not get_accelerator().device_name().startswith("cuda"): warnings.warn( "Skip modifying optimizer as device is not supported, " - "device name: {}".format(get_accelerator().device_name()), UserWarning, + "device name: {}".format(get_accelerator().device_name()), + UserWarning, ) return False