Skip to content

Commit

Permalink
Fix random seed usage and handle empty descriptions
Browse files Browse the repository at this point in the history
  • Loading branch information
alekszievr committed Jan 21, 2025
1 parent 1c16a17 commit 9fec8fd
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 3 deletions.
2 changes: 1 addition & 1 deletion evals/eval_on_hotpot.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
from pathlib import Path

logger = logging.getLogger(__name__)
random.seed(42)


async def answer_qa_instance(instance, context_provider, contexts_filename):
Expand Down Expand Up @@ -110,6 +109,7 @@ async def eval_on_QA_dataset(
dataset = load_qa_dataset(dataset_name_or_filename)
context_provider = qa_context_providers[context_provider_name]
eval_metrics = get_metrics(metric_name_list)
random.seed(42)
instances = dataset if not num_samples else random.sample(dataset, num_samples)

contexts_filename = Path(out_path) / Path(
Expand Down
16 changes: 14 additions & 2 deletions evals/qa_context_provider_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,22 @@ def _insight_to_string(triplet: tuple) -> str:
return ""

node1_name = node1["name"] if "name" in node1 else "N/A"
node1_description = node1["description"] if "description" in node1 else node1["text"]
node1_description = (
node1["description"]
if "description" in node1
else node1["text"]
if "text" in node1
else "N/A"
)
node1_string = f"name: {node1_name}, description: {node1_description}"
node2_name = node2["name"] if "name" in node2 else "N/A"
node2_description = node2["description"] if "description" in node2 else node2["text"]
node2_description = (
node2["description"]
if "description" in node2
else node2["text"]
if "text" in node2
else "N/A"
)
node2_string = f"name: {node2_name}, description: {node2_description}"

edge_string = edge.get("relationship_name", "")
Expand Down

0 comments on commit 9fec8fd

Please sign in to comment.