[PEFT] Pass token when calling find_adapter_config (#26488)

* try

* nit

* nits

src/transformers/modeling_utils.py

@@ -2480,6 +2480,9 @@ def from_pretrained(
                 )
             token = use_auth_token
 
+        if token is not None and "token" not in adapter_kwargs:
+            adapter_kwargs["token"] = token
+
         if use_safetensors is None and not is_safetensors_available():
             use_safetensors = False
 
@@ -2526,7 +2529,6 @@ def from_pretrained(
                     resume_download=resume_download,
                     proxies=proxies,
                     local_files_only=local_files_only,
-                    token=token,
                     _commit_hash=commit_hash,
                     **adapter_kwargs,
                 )

src/transformers/models/auto/auto_factory.py

@@ -498,6 +498,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
         if is_peft_available():
             if adapter_kwargs is None:
                 adapter_kwargs = {}
+                if token is not None:
+                    adapter_kwargs["token"] = token
 
             maybe_adapter_path = find_adapter_config_file(
                 pretrained_model_name_or_path, _commit_hash=commit_hash, **adapter_kwargs