fix faithfulness template

Signed-off-by: lilacheden <[email protected]>

prepare/templates/rag_eval/rag_eval_numeric.py

@@ -124,6 +124,7 @@ def add_rag_templates(
 
 faithfilness_instructions_with_question_simplified_category = """You are given a question, the corresponding evidence and a prediction from a model. Compare the "Prediction" and the "Evidence" to determine to what extent the prediction is grounded in the evidence.
 To be grounded in the evidence, all the information of the prediction must either be present in the evidence or deducible from the evidence.\n
+Base your answer only on the information in the evidence. If the prediction is correct but not present in the evidence - it is not grounded.
 The question is only given for context, and is irrelevant for determining the groundedness of the prediction.
 Reply with one of the 4 options, without any further explanations:
 "Completely Grounded" - if the prediction is grounded in the evidence.
@@ -134,6 +135,7 @@ def add_rag_templates(
 
 faithfulness_instructions_no_question_simplified_category = """You are given a grounding evidence and a prediction from a model. Compare the "Prediction" and the "Evidence" to determine to what extent the prediction is grounded in the evidence.
 To be grounded in the evidence, all the information of the prediction must either be present in the evidence or deducible from the evidence.\n
+Base your answer only on the information in the evidence. If the prediction is correct but not present in the evidence than it is not grounded.
 Reply with one of the 4 options, without any further explanations:
 "Completely Grounded" - if the prediction is grounded in the evidence.
 "Mostly grounded" - if the vast majority of the information in the prediction is grounded in the evidence, but there is a small or negligible part of the prediction which is not present in the evidence.
@@ -144,13 +146,15 @@ def add_rag_templates(
 faithfilness_instructions_with_question_simplified_verbal_good_bad = """You are given a question, the corresponding evidence and a prediction from a model.
 Compare the "Prediction" and the "Evidence" to determine whether or not the prediction is grounded in the evidence.
 A good prediction is a prediction which is grounded in the evidence, i.e. all the information of the prediction must either be present in the evidence or deducible from the evidence.\n
+Base your answer only on the information in the evidence. If the prediction is correct but not present in the evidence - it is not grounded.
 The question is only given for context, and is irrelevant for determining the groundedness of the prediction.
 Provide a rating from one of the following choices: 'Very Bad', 'Bad', 'Mediocre', 'Good', 'Very Good'. Reply using the format of [[rating]], for example: '[[Mediocre]]'.
 """
 
 faithfulness_instructions_no_question_simplified_verbal_good_bad = """You are given a grounding evidence and a prediction from a model.
 Compare the "Prediction" and the "Evidence" to determine whether or not the prediction is grounded in the evidence.
 A good prediction is a prediction which is grounded in the evidence, i.e. all the information of the prediction must either be present in the evidence or deducible from the evidence.\n
+Base your answer only on the information in the evidence. If the prediction is correct but not present in the evidence - it is not grounded.
 Provide a rating from one of the following choices: 'Very Bad', 'Bad', 'Mediocre', 'Good', 'Very Good'. Reply using the format of [[rating]], for example: '[[Mediocre]]'.
 """
 

src/unitxt/catalog/templates/rag_eval/faithfulness/judge_no_question_simplified_verbal.json

@@ -9,5 +9,5 @@
     ],
     "reference": "{number_val}",
     "target_prefix": "Answer: ",
-    "instruction": "You are given a grounding evidence and a prediction from a model. Compare the \"Prediction\" and the \"Evidence\" to determine to what extent the prediction is grounded in the evidence.\nTo be grounded in the evidence, all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nReply with one of the 4 options, without any further explanations:\n\"Completely Grounded\" - if the prediction is grounded in the evidence.\n\"Mostly grounded\" - if the vast majority of the information in the prediction is grounded in the evidence, but there is a small or negligible part of the prediction which is not present in the evidence.\n\"Somewhat grounded\" - If some of the information in the prediction is grounded in the evidence.\n\"Not grounded\" - If most or all of the information in the prediction is not grounded in the evidence\n"
+    "instruction": "You are given a grounding evidence and a prediction from a model. Compare the \"Prediction\" and the \"Evidence\" to determine to what extent the prediction is grounded in the evidence.\nTo be grounded in the evidence, all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nBase your answer only on the information in the evidence. If the prediction is correct but not present in the evidence than it is not grounded.\nReply with one of the 4 options, without any further explanations:\n\"Completely Grounded\" - if the prediction is grounded in the evidence.\n\"Mostly grounded\" - if the vast majority of the information in the prediction is grounded in the evidence, but there is a small or negligible part of the prediction which is not present in the evidence.\n\"Somewhat grounded\" - If some of the information in the prediction is grounded in the evidence.\n\"Not grounded\" - If most or all of the information in the prediction is not grounded in the evidence\n"
 }

src/unitxt/catalog/templates/rag_eval/faithfulness/judge_no_question_simplified_verbal_good_bad.json

@@ -9,5 +9,5 @@
     ],
     "reference": "{number_val}",
     "target_prefix": "Answer: ",
-    "instruction": "You are given a grounding evidence and a prediction from a model.\nCompare the \"Prediction\" and the \"Evidence\" to determine whether or not the prediction is grounded in the evidence.\nA good prediction is a prediction which is grounded in the evidence, i.e. all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nProvide a rating from one of the following choices: 'Very Bad', 'Bad', 'Mediocre', 'Good', 'Very Good'. Reply using the format of [[rating]], for example: '[[Mediocre]]'.\n"
+    "instruction": "You are given a grounding evidence and a prediction from a model.\nCompare the \"Prediction\" and the \"Evidence\" to determine whether or not the prediction is grounded in the evidence.\nA good prediction is a prediction which is grounded in the evidence, i.e. all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nBase your answer only on the information in the evidence. If the prediction is correct but not present in the evidence - it is not grounded.\nProvide a rating from one of the following choices: 'Very Bad', 'Bad', 'Mediocre', 'Good', 'Very Good'. Reply using the format of [[rating]], for example: '[[Mediocre]]'.\n"
 }

src/unitxt/catalog/templates/rag_eval/faithfulness/judge_with_question_simplified_verbal.json

@@ -9,5 +9,5 @@
     ],
     "reference": "{number_val}",
     "target_prefix": "Answer: ",
-    "instruction": "You are given a question, the corresponding evidence and a prediction from a model. Compare the \"Prediction\" and the \"Evidence\" to determine to what extent the prediction is grounded in the evidence.\nTo be grounded in the evidence, all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nThe question is only given for context, and is irrelevant for determining the groundedness of the prediction.\nReply with one of the 4 options, without any further explanations:\n\"Completely Grounded\" - if the prediction is grounded in the evidence.\n\"Mostly grounded\" - if the vast majority of the information in the prediction is grounded in the evidence, but there is a small or negligible part of the prediction which is not present in the evidence.\n\"Somewhat grounded\" - If some of the information in the prediction is grounded in the evidence.\n\"Not grounded\" - If most or all of the information in the prediction is not grounded in the evidence\n"
+    "instruction": "You are given a question, the corresponding evidence and a prediction from a model. Compare the \"Prediction\" and the \"Evidence\" to determine to what extent the prediction is grounded in the evidence.\nTo be grounded in the evidence, all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nBase your answer only on the information in the evidence. If the prediction is correct but not present in the evidence - it is not grounded.\nThe question is only given for context, and is irrelevant for determining the groundedness of the prediction.\nReply with one of the 4 options, without any further explanations:\n\"Completely Grounded\" - if the prediction is grounded in the evidence.\n\"Mostly grounded\" - if the vast majority of the information in the prediction is grounded in the evidence, but there is a small or negligible part of the prediction which is not present in the evidence.\n\"Somewhat grounded\" - If some of the information in the prediction is grounded in the evidence.\n\"Not grounded\" - If most or all of the information in the prediction is not grounded in the evidence\n"
 }

src/unitxt/catalog/templates/rag_eval/faithfulness/judge_with_question_simplified_verbal_good_bad.json

@@ -9,5 +9,5 @@
     ],
     "reference": "{number_val}",
     "target_prefix": "Answer: ",
-    "instruction": "You are given a question, the corresponding evidence and a prediction from a model.\nCompare the \"Prediction\" and the \"Evidence\" to determine whether or not the prediction is grounded in the evidence.\nA good prediction is a prediction which is grounded in the evidence, i.e. all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nThe question is only given for context, and is irrelevant for determining the groundedness of the prediction.\nProvide a rating from one of the following choices: 'Very Bad', 'Bad', 'Mediocre', 'Good', 'Very Good'. Reply using the format of [[rating]], for example: '[[Mediocre]]'.\n"
+    "instruction": "You are given a question, the corresponding evidence and a prediction from a model.\nCompare the \"Prediction\" and the \"Evidence\" to determine whether or not the prediction is grounded in the evidence.\nA good prediction is a prediction which is grounded in the evidence, i.e. all the information of the prediction must either be present in the evidence or deducible from the evidence.\n\nBase your answer only on the information in the evidence. If the prediction is correct but not present in the evidence - it is not grounded.\nThe question is only given for context, and is irrelevant for determining the groundedness of the prediction.\nProvide a rating from one of the following choices: 'Very Bad', 'Bad', 'Mediocre', 'Good', 'Very Good'. Reply using the format of [[rating]], for example: '[[Mediocre]]'.\n"
 }