Skip to content

Commit

Permalink
refactor: Generate eval result in separate method (#5001)
Browse files Browse the repository at this point in the history
Co-authored-by: Silvano Cerza <[email protected]>
  • Loading branch information
bogdankostic and silvanocerza authored May 25, 2023
1 parent 44fd0cf commit 19829da
Showing 1 changed file with 34 additions and 4 deletions.
38 changes: 34 additions & 4 deletions haystack/pipelines/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1366,15 +1366,45 @@ def eval_batch(
event_properties={"pipeline.classname": self.__class__.__name__, "pipeline.config_hash": self.config_hash},
)

predictions_batches = self.run_batch(
queries=[label.query for label in labels], labels=labels, documents=documents, params=params, debug=True
)

eval_result = self._generate_eval_result_from_batch_preds(
predictions_batches=predictions_batches,
params=params,
sas_model_name_or_path=sas_model_name_or_path,
sas_batch_size=sas_batch_size,
sas_use_gpu=sas_use_gpu,
add_isolated_node_eval=add_isolated_node_eval,
custom_document_id_field=custom_document_id_field,
context_matching_min_length=context_matching_min_length,
context_matching_boost_split_overlaps=context_matching_boost_split_overlaps,
context_matching_threshold=context_matching_threshold,
use_auth_token=use_auth_token,
)

return eval_result

def _generate_eval_result_from_batch_preds(
self,
predictions_batches: Dict,
params: Optional[dict] = None,
sas_model_name_or_path: Optional[str] = None,
sas_batch_size: int = 32,
sas_use_gpu: bool = True,
add_isolated_node_eval: bool = False,
custom_document_id_field: Optional[str] = None,
context_matching_min_length: int = 100,
context_matching_boost_split_overlaps: bool = True,
context_matching_threshold: float = 65.0,
use_auth_token: Optional[Union[str, bool]] = None,
) -> EvaluationResult:
eval_result = EvaluationResult()
if add_isolated_node_eval:
params = {} if params is None else params.copy()
params["add_isolated_node_eval"] = True

predictions_batches = self.run_batch(
queries=[label.query for label in labels], labels=labels, documents=documents, params=params, debug=True
)

for node_name in predictions_batches["_debug"].keys():
node_output = predictions_batches["_debug"][node_name]["output"]
df = self._build_eval_dataframe(
Expand Down

0 comments on commit 19829da

Please sign in to comment.