diff --git a/scripts/ci/e2e_benchmark.py b/scripts/ci/e2e_benchmark.py index 02a0b59b81..e928c720bd 100644 --- a/scripts/ci/e2e_benchmark.py +++ b/scripts/ci/e2e_benchmark.py @@ -86,9 +86,9 @@ def bootstrap_confidence_interval(data, num_samples=1000, confidence_level=0.95) mean = np.mean(means) return mean, lower_bound, upper_bound -def calculate_confidence_interval(data): +def calculate_confidence_interval(data, min_is_best=True): mean, lower, upper = bootstrap_confidence_interval(data) - min_value = np.min(data) + min_value = np.min(data) if min_is_best else np.max(data) return mean, (upper - lower) / 2, min_value @@ -117,7 +117,7 @@ def main(): total_time, total_ci, total_best = calculate_confidence_interval(np.sum(all_times, axis=1)) - ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1)) + ops_per_sec, ops_per_sec_ci, ops_per_sec_best = calculate_confidence_interval(float(all_times.shape[1]) / np.sum(all_times / 1000, axis=1), min_is_best=False) min_time, min_ci, _ = calculate_confidence_interval(np.min(all_times, axis=1)) mean_time, mean_ci, _ = calculate_confidence_interval(np.mean(all_times, axis=1)) median_time, median_ci, _ = calculate_confidence_interval(np.median(all_times, axis=1)) diff --git a/src/benchmarks/bench.cpp b/src/benchmarks/bench.cpp index b1d88aa0e6..39a941ed99 100644 --- a/src/benchmarks/bench.cpp +++ b/src/benchmarks/bench.cpp @@ -134,6 +134,7 @@ struct ConfidenceInterval double mean; double confidence; double min; + double max; }; // Helper function to calculate the bootstrap confidence interval @@ -161,8 +162,10 @@ ConfidenceInterval confidenceInterval(const std::vector &data, double upper_bound = means[(int)((1 + confidence_level) / 2 * num_samples)]; double mean = std::accumulate(means.begin(), means.end(), 0.0) / means.size(); - ConfidenceInterval ci = { - mean, (upper_bound - lower_bound) / 2, *std::min_element(data.begin(), data.end())}; + ConfidenceInterval ci = {mean, + (upper_bound - lower_bound) / 2, + *std::min_element(data.begin(), data.end()), + *std::max_element(data.begin(), data.end())}; return ci; } @@ -260,7 +263,7 @@ std::ostream &operator<<(std::ostream &os, Statistics &statistics) ConfidenceInterval ops_ci = statistics.ops_per_sec(); os << "ops: " << ops_ci.mean << " ± " << ops_ci.confidence << " ops/s. " - << "best: " << ops_ci.min << "ops/s." << std::endl; + << "best: " << ops_ci.max << "ops/s." << std::endl; os << "total: " << total_ci.mean << " ± " << total_ci.confidence << "ms. " << "best: " << total_ci.min << "ms." << std::endl; os << "avg: " << mean_ci.mean << " ± " << mean_ci.confidence << "ms" << std::endl;