ensure best record is never worse than worst record on any metric

Signed-off-by: Shehzeen Hussain <[email protected]>
paarthneekhara · Jan 28, 2025 · b029d92 · b029d92
1 parent 42cde9e
commit b029d92
Showing 1 changed file with 6 additions and 7 deletions.
diff --git a/scripts/t5tts/dpo/create_preference_pairs.py b/scripts/t5tts/dpo/create_preference_pairs.py
@@ -194,22 +194,21 @@ def create_chosen_rejected_records(records_orig, group_size=6, num_chosen_per_gr
         for sidx, record in enumerate(group):
             cer_sim_indices.append((record['cer_gts'], record['pred_context_similarity'], sidx))
 
+        cer_sim_indices_orig = copy.deepcopy(cer_sim_indices)
         cer_sim_indices = pareto_rank(cer_sim_indices)
 
         for cgi in chosen_group_indices:
             for rji in rejected_group_indices: 
                 best_record = group[cer_sim_indices[cgi][3]]
                 worst_record = group[cer_sim_indices[rji][3]]
                 best_record['reward'] = 1
-                if worst_record['pred_context_similarity'] > best_record['pred_context_similarity']:
-                    reward_delta = (worst_record['cer_gts'] - best_record['cer_gts'])
-                else:
-                    reward_delta = (worst_record['cer_gts'] - best_record['cer_gts']) + (best_record['pred_context_similarity'] - worst_record['pred_context_similarity'])
-
-                if reward_delta <= 0 or worst_record['cer_gts'] < best_record['cer_gts']:
+                reward_delta = (worst_record['cer_gts'] - best_record['cer_gts']) + (best_record['pred_context_similarity'] - worst_record['pred_context_similarity'])
+                if reward_delta <= 0 or worst_record['cer_gts'] < best_record['cer_gts'] or worst_record['pred_context_similarity'] > best_record['pred_context_similarity']:
                     print("Warning reward_delta is not positive", reward_delta, best_record['cer_gts'], worst_record['cer_gts'], best_record['pred_context_similarity'], worst_record['pred_context_similarity'])
+                    print(cer_sim_indices_orig)
+                    print(cer_sim_indices)
                 else:
-                    # Never add pairs in which rejected has better CER than chosen
+                    # Never add pairs in which rejected has better CER than chosen or better context similarity
                     reward_delta = max(0.001, reward_delta)
                     worst_record['reward'] = 1.0 - reward_delta
                     best_records.append(best_record)