Errors during the training!

[merecesarchviz]

During training i always get this errors! any idea what it and how i can solve it?

Training Environment:
| > Backend: Torch
| > Mixed precision: False
| > Precision: float32
| > Current device: 0
| > Num. of GPUs: 1
| > Num. of CPUs: 48
| > Num. of Torch Threads: 1
| > Torch seed: 1
| > Torch CUDNN: True
| > Torch CUDNN deterministic: False
| > Torch CUDNN benchmark: False
| > Torch TF32 MatMul: False
Start Tensorboard: tensorboard --logdir=F:\Ai\Clone_Voices\xtts-finetune-webui\finetune_models\run\training\GPT_XTTS_FT-August-08-2024_10+10AM-abf3ed9

Model has 517360175 parameters

EPOCH: 0/6
--> F:\Ai\Clone_Voices\xtts-finetune-webui\finetune_models\run\training\GPT_XTTS_FT-August-08-2024_10+10AM-abf3ed9

TRAINING (2024-08-08 10:10:45)
Traceback (most recent call last):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1132, in _try_get_data
data = self._data_queue.get(timeout=timeout)
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\multiprocessing\queues.py", line 114, in get
raise Empty
_queue.Empty

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1605, in fit
self._fit()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1557, in _fit
self.train_epoch()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1272, in train_epoch
for cur_step, batch in enumerate(self.train_loader):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 630, in next
data = self._next_data()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1328, in _next_data
idx, data = self._get_data()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1294, in _get_data
success, data = self._try_get_data()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1145, in _try_get_data
raise RuntimeError(f'DataLoader worker (pid(s) {pids_str}) exited unexpectedly') from e
RuntimeError: DataLoader worker (pid(s) 36108, 21432, 26540, 35352) exited unexpectedly

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\xtts_demo.py", line 381, in train_model
speaker_xtts_path,config_path, original_xtts_checkpoint, vocab_file, exp_path, speaker_wav = train_gpt(custom_model,version,language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv, output_path=output_path, max_audio_length=max_audio_length)
File "F:\Ai\Clone_Voices\xtts-finetune-webui\utils\gpt_train.py", line 198, in train_gpt
trainer.fit()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1632, in fit
remove_experiment_folder(self.output_path)
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\generic_utils.py", line 78, in remove_experiment_folder
fs.rm(experiment_path, recursive=True)
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\fsspec\implementations\local.py", line 172, in rm
shutil.rmtree(p)
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\shutil.py", line 750, in rmtree
return _rmtree_unsafe(path, onerror)
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\shutil.py", line 620, in _rmtree_unsafe
onerror(os.unlink, fullname, sys.exc_info())
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\shutil.py", line 618, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] O processo não pode aceder ao ficheiro porque este está a ser utilizado por outro processo: 'F:/Ai/Clone_Voices/xtts-finetune-webui/finetune_models/run/training/GPT_XTTS_FT-August-08-2024_10+10AM-abf3ed9\trainer_0_log.txt'
Traceback (most recent call last):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\gradio\queueing.py", line 489, in call_prediction
output = await route_utils.call_process_api(
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\gradio\route_utils.py", line 232, in call_process_api
output = await app.get_blocks().process_api(
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\gradio\blocks.py", line 1570, in process_api
data = self.postprocess_data(fn_index, result["prediction"], state)
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\gradio\blocks.py", line 1397, in postprocess_data
self.validate_outputs(fn_index, predictions) # type: ignore
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\gradio\blocks.py", line 1371, in validate_outputs
raise ValueError(
ValueError: An event handler (train_model) didn't receive enough output values (needed: 6, received: 5).
Wanted outputs:
[label, textbox, textbox, textbox, textbox, textbox]
Received outputs:
["The training was interrupted due an error !! Please check the console to check the full error message!
Error summary: Traceback (most recent call last):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1132, in _try_get_data
data = self._data_queue.get(timeout=timeout)
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\multiprocessing\queues.py", line 114, in get
raise Empty
_queue.Empty

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1605, in fit
self._fit()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1557, in _fit
self.train_epoch()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1272, in train_epoch
for cur_step, batch in enumerate(self.train_loader):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 630, in next
data = self._next_data()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1328, in _next_data
idx, data = self._get_data()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1294, in _get_data
success, data = self._try_get_data()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\torch\utils\data\dataloader.py", line 1145, in _try_get_data
raise RuntimeError(f'DataLoader worker (pid(s) {pids_str}) exited unexpectedly') from e
RuntimeError: DataLoader worker (pid(s) 36108, 21432, 26540, 35352) exited unexpectedly

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "F:\Ai\Clone_Voices\xtts-finetune-webui\xtts_demo.py", line 381, in train_model
speaker_xtts_path,config_path, original_xtts_checkpoint, vocab_file, exp_path, speaker_wav = train_gpt(custom_model,version,language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv, output_path=output_path, max_audio_length=max_audio_length)
File "F:\Ai\Clone_Voices\xtts-finetune-webui\utils\gpt_train.py", line 198, in train_gpt
trainer.fit()
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\trainer.py", line 1632, in fit
remove_experiment_folder(self.output_path)
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\trainer\generic_utils.py", line 78, in remove_experiment_folder
fs.rm(experiment_path, recursive=True)
File "F:\Ai\Clone_Voices\xtts-finetune-webui\venv\lib\site-packages\fsspec\implementations\local.py", line 172, in rm
shutil.rmtree(p)
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\shutil.py", line 750, in rmtree
return _rmtree_unsafe(path, onerror)
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\shutil.py", line 620, in _rmtree_unsafe
onerror(os.unlink, fullname, sys.exc_info())
File "C:\Users\Ryzen_Reaper\AppData\Local\Programs\Python\Python310\lib\shutil.py", line 618, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] O processo não pode aceder ao ficheiro porque este está a ser utilizado por outro processo: 'F:/Ai/Clone_Voices/xtts-finetune-webui/finetune_models/run/training/GPT_XTTS_FT-August-08-2024_10+10AM-abf3ed9\trainer_0_log.txt'
", "", "", "", ""]