From cc7cebd50382985ba5a62c548f7b0f7fbf44e2cd Mon Sep 17 00:00:00 2001 From: Aasavari Kakne Date: Sat, 2 Nov 2024 02:02:12 -0700 Subject: [PATCH] Ragaaf - adding new metric 'context recall' (#184) * small fix for ragas.py Signed-off-by: aasavari * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed error when metrics arg is used Signed-off-by: aasavari * updated README Signed-off-by: aasavari * added key features Signed-off-by: aasavari * edited formatting Signed-off-by: aasavari * improved readability Signed-off-by: aasavari * improved note in model section Signed-off-by: aasavari * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * spell check Signed-off-by: aasavari * adding context recall Signed-off-by: aasavari * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: aasavari Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../ragaaf/prompt_templates/context_recall.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 evals/metrics/ragaaf/prompt_templates/context_recall.py diff --git a/evals/metrics/ragaaf/prompt_templates/context_recall.py b/evals/metrics/ragaaf/prompt_templates/context_recall.py new file mode 100644 index 00000000..78c91390 --- /dev/null +++ b/evals/metrics/ragaaf/prompt_templates/context_recall.py @@ -0,0 +1,13 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +class ContextRecall: + name = "context_recall" + required_columns = ["context", "ground_truth"] + template = """- Context Recall: context recall measures how well ground truth is supported by the context. + - Score 1: None of facts/steps/aspects in the ground truth are supported by the context. + - Score 2: Only a small part of facts/steps/aspects in the ground truth are supported by the context. + - Score 3: About half of facts/steps/aspects in the ground truth are supported by the context. + - Score 4: All main/important facts/steps/aspects in the ground truth are supported by the context but some of the details are not supported by the context. + - Score 5: All of facts/steps/aspects in the ground truth are supported by the context. """