From f7297a1dee21adb796b077bada3b551af3b902cc Mon Sep 17 00:00:00 2001 From: "Artem M. Chirkin" <9253178+achirkin@users.noreply.github.com> Date: Fri, 29 Jul 2022 21:11:50 +0200 Subject: [PATCH] Relax ivf-flat test recall thresholds (#766) When `n_probes = n_lists`, ivf-flat should perform equal to the brute-force kNN (`recall == 1` in tests). However, it looks like in some rare cases in CI the recall is slightly smaller. This could be due to some numeric errors or due to the radix_topk not being stable. This PR reduces the recall threshold in the tests for now to avoid occasional failures in the main branch while the issue is investigated. Authors: - Artem M. Chirkin (https://github.com/achirkin) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/raft/pull/766 --- cpp/test/spatial/ann_ivf_flat.cu | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/test/spatial/ann_ivf_flat.cu b/cpp/test/spatial/ann_ivf_flat.cu index 7468fd75b7..a2398e96fc 100644 --- a/cpp/test/spatial/ann_ivf_flat.cu +++ b/cpp/test/spatial/ann_ivf_flat.cu @@ -94,8 +94,12 @@ auto eval_knn(const std::vector& expected_idx, RAFT_LOG_INFO("Recall = %zu/%zu", match_count, total_count); double actual_recall = static_cast(match_count) / static_cast(total_count); if (actual_recall < min_recall - eps) { - RAFT_LOG_WARN("Recall is suspiciously too low (%f < %f)", actual_recall, min_recall); - if (match_count == 0 || actual_recall < min_recall * min_recall - eps) { + if (actual_recall < min_recall * min_recall - eps) { + RAFT_LOG_ERROR("Recall is much lower than the minimum (%f < %f)", actual_recall, min_recall); + } else { + RAFT_LOG_WARN("Recall is suspiciously too low (%f < %f)", actual_recall, min_recall); + } + if (match_count == 0 || actual_recall < min_recall * std::min(min_recall, 0.5) - eps) { return testing::AssertionFailure() << "actual recall (" << actual_recall << ") is much smaller than the minimum expected recall (" << min_recall << ").";