[LoRA] fix adapter movement when using DoRA. (huggingface#9411)

fix adapter movement when using DoRA.
UmerHA · Sep 13, 2024 · 6dc6486 · 6dc6486
1 parent 1e8cf27
commit 6dc6486
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py
@@ -699,9 +699,10 @@ def set_lora_device(self, adapter_names: List[str], device: Union[torch.device,
                             module.lora_B[adapter_name].to(device)
                             # this is a param, not a module, so device placement is not in-place -> re-assign
                             if hasattr(module, "lora_magnitude_vector") and module.lora_magnitude_vector is not None:
-                                module.lora_magnitude_vector[adapter_name] = module.lora_magnitude_vector[
-                                    adapter_name
-                                ].to(device)
+                                if adapter_name in module.lora_magnitude_vector:
+                                    module.lora_magnitude_vector[adapter_name] = module.lora_magnitude_vector[
+                                        adapter_name
+                                    ].to(device)
 
     @staticmethod
     def pack_weights(layers, prefix):