Skip to content

Commit

Permalink
fix: Use add_isolated_node_eval of eval_batch in run_batch (#5223)
Browse files Browse the repository at this point in the history
* Fix isolated node eval in eval_batch

* Add unit test
bogdankostic authored and anakin87 committed Jun 28, 2023
1 parent 503eee3 commit 42d2e55
Showing 2 changed files with 15 additions and 7 deletions.
10 changes: 3 additions & 7 deletions haystack/pipelines/base.py
Original file line number Diff line number Diff line change
@@ -1366,18 +1366,19 @@ def eval_batch(
event_name="Evaluation",
event_properties={"pipeline.classname": self.__class__.__name__, "pipeline.config_hash": self.config_hash},
)
if add_isolated_node_eval:
params = {} if params is None else params.copy()
params["add_isolated_node_eval"] = True

predictions_batches = self.run_batch(
queries=[label.query for label in labels], labels=labels, documents=documents, params=params, debug=True
)

eval_result = self._generate_eval_result_from_batch_preds(
predictions_batches=predictions_batches,
params=params,
sas_model_name_or_path=sas_model_name_or_path,
sas_batch_size=sas_batch_size,
sas_use_gpu=sas_use_gpu,
add_isolated_node_eval=add_isolated_node_eval,
custom_document_id_field=custom_document_id_field,
context_matching_min_length=context_matching_min_length,
context_matching_boost_split_overlaps=context_matching_boost_split_overlaps,
@@ -1390,21 +1391,16 @@ def eval_batch(
def _generate_eval_result_from_batch_preds(
self,
predictions_batches: Dict,
params: Optional[dict] = None,
sas_model_name_or_path: Optional[str] = None,
sas_batch_size: int = 32,
sas_use_gpu: bool = True,
add_isolated_node_eval: bool = False,
custom_document_id_field: Optional[str] = None,
context_matching_min_length: int = 100,
context_matching_boost_split_overlaps: bool = True,
context_matching_threshold: float = 65.0,
use_auth_token: Optional[Union[str, bool]] = None,
) -> EvaluationResult:
eval_result = EvaluationResult()
if add_isolated_node_eval:
params = {} if params is None else params.copy()
params["add_isolated_node_eval"] = True

for node_name in predictions_batches["_debug"].keys():
node_output = predictions_batches["_debug"][node_name]["output"]
12 changes: 12 additions & 0 deletions test/pipelines/test_eval_batch.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import logging
from unittest.mock import patch

import pytest
import sys
from copy import deepcopy
@@ -21,6 +23,16 @@
from haystack.schema import Answer, Document, EvaluationResult, Label, MultiLabel, Span


@pytest.mark.unit
@patch("haystack.pipelines.base.Pipeline.run_batch")
def test_eval_batch_add_isolated_node_eval_passed_to_run_batch(mock_run_batch):
pipeline = Pipeline()
pipeline.eval_batch(labels=EVAL_LABELS, add_isolated_node_eval=True)
_, kwargs = mock_run_batch.call_args
assert "add_isolated_node_eval" in kwargs["params"]
assert kwargs["params"]["add_isolated_node_eval"] is True


@pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Causes OOM on windows github runner")
@pytest.mark.parametrize("document_store_with_docs", ["memory"], indirect=True)
@pytest.mark.parametrize("retriever_with_docs", ["embedding"], indirect=True)

0 comments on commit 42d2e55

Please sign in to comment.