From b1a03d8e5927069af4987d5c9fd200f71384f6dd Mon Sep 17 00:00:00 2001 From: Abhigyan Acherjee Date: Sun, 3 Mar 2024 22:16:12 -0500 Subject: [PATCH] [RF] Added benchmark plot scripts and update RooFitBinned benchmarks --- root/roofit/roofit/CMakeLists.txt | 2 + root/roofit/roofit/RooFitBinnedBenchmarks.cxx | 106 ++++++++++-------- root/roofit/roofit/compare_benchmarks.py | 52 +++++++++ root/roofit/roofit/run_benchmarks.sh | 27 +++++ 4 files changed, 138 insertions(+), 49 deletions(-) create mode 100644 root/roofit/roofit/compare_benchmarks.py create mode 100755 root/roofit/roofit/run_benchmarks.sh diff --git a/root/roofit/roofit/CMakeLists.txt b/root/roofit/roofit/CMakeLists.txt index ae3956fe8..3341bd26e 100644 --- a/root/roofit/roofit/CMakeLists.txt +++ b/root/roofit/roofit/CMakeLists.txt @@ -19,6 +19,8 @@ if(cuda) endif() file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/benchRooFitBackends_make_plot.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/compare_benchmarks.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_benchmarks.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) RB_ADD_GBENCHMARK(benchCodeSquashAD benchCodeSquashAD.cxx diff --git a/root/roofit/roofit/RooFitBinnedBenchmarks.cxx b/root/roofit/roofit/RooFitBinnedBenchmarks.cxx index 809effbb3..4d06130ce 100644 --- a/root/roofit/roofit/RooFitBinnedBenchmarks.cxx +++ b/root/roofit/roofit/RooFitBinnedBenchmarks.cxx @@ -19,25 +19,27 @@ using namespace RooStats; using namespace HistFactory; namespace { - constexpr bool verbose = false; +constexpr bool verbose = false; - // test matrix configuration - const std::vector nChannelsVector = {1, 2, 3}; - const std::vector nBinsVector {5, 10, 15}; - const int nBinsForChannelScan = 10; - const int nChannelsForBinScan = 1; - const std::vector nCPUVector {1, 2, 3}; +// test matrix configuration +const std::vector nChannelsVector = {1, 2, 3}; +const std::vector nBinsVector{5, 10, 15}; +const int nBinsForChannelScan = 10; +const int nChannelsForBinScan = 1; +const std::vector nCPUVector{1}; - constexpr auto evalBackend = RooFit::EvalBackend::Value::Cpu; +////default evaluation backend +std::string evalBackend = "cpu"; - auto const timeUnit = benchmark::kMillisecond; +auto const timeUnit = benchmark::kMillisecond; - void setupRooMsgService() { - RooMsgService::instance().setGlobalKillBelow(RooFit::FATAL); - RooMsgService::instance().getStream(1).removeTopic(RooFit::Minimization); - RooMsgService::instance().getStream(1).removeTopic(RooFit::NumIntegration); - RooMsgService::instance().getStream(1).removeTopic(RooFit::Eval); - } +void setupRooMsgService() +{ + RooMsgService::instance().setGlobalKillBelow(RooFit::FATAL); + RooMsgService::instance().getStream(1).removeTopic(RooFit::Minimization); + RooMsgService::instance().getStream(1).removeTopic(RooFit::NumIntegration); + RooMsgService::instance().getStream(1).removeTopic(RooFit::Eval); +} } // namespace @@ -48,9 +50,9 @@ Sample addVariations(Sample asample, int nnps, bool channel_crosstalk, int chann Double_t random = R->Rndm(); double uncertainty_up = (1 + random) / sqrt(100); double uncertainty_down = (1 - random) / sqrt(100); - if(verbose) { - std::cout << "in channel " << channel << "nuisance +/- [" << uncertainty_up << "," << uncertainty_down << "]" - << std::endl; + if (verbose) { + std::cout << "in channel " << channel << "nuisance +/- [" << uncertainty_up << "," << uncertainty_down << "]" + << std::endl; } std::string nuis_name = "norm_uncertainty_" + std::to_string(nuis); if (!channel_crosstalk) { @@ -101,8 +103,8 @@ std::unique_ptr makeChannel(int channel, int nbi void buildBinnedTest(int n_channels = 1, int nbins = 10, int nnps = 1, const char *name_rootfile = "") { - if(verbose) { - std::cout << "in build binned test with output" << name_rootfile << std::endl; + if (verbose) { + std::cout << "in build binned test with output" << name_rootfile << std::endl; } Measurement meas("meas", "meas"); meas.SetPOI("SignalStrength"); @@ -121,19 +123,20 @@ void buildBinnedTest(int n_channels = 1, int nbins = 10, int nnps = 1, const cha } else { ws = std::unique_ptr{hist2workspace.MakeCombinedModel(meas)}; } - for (RooAbsArg * arg : ws->components()) { + for (RooAbsArg *arg : ws->components()) { if (arg->IsA() == RooRealSumPdf::Class()) { arg->setAttribute("BinnedLikelihood"); - if(verbose) std::cout << "component " << arg->GetName() << " is a binned likelihood" << std::endl; + if (verbose) + std::cout << "component " << arg->GetName() << " is a binned likelihood" << std::endl; } } ws->SetName("BinnedWorkspace"); ws->writeToFile(name_rootfile); } -//############## End of Base Algorithms ############################## -//#################################################################### -//############## Start Of # Tests ############################# +// ############## End of Base Algorithms ############################## +// #################################################################### +// ############## Start Of # Tests ############################# static void BM_RooFit_BinnedTestMigrad(benchmark::State &state) { @@ -145,7 +148,8 @@ static void BM_RooFit_BinnedTestMigrad(benchmark::State &state) auto infile = std::make_unique("workspace.root", "RECREATE"); // if (infile->IsZombie()) { buildBinnedTest(chan, nbins, 2, "workspace.root"); - if(verbose) std::cout << "Workspace for tests was created!" << std::endl; + if (verbose) + std::cout << "Workspace for tests was created!" << std::endl; //} infile.reset(TFile::Open("workspace.root")); RooWorkspace *w = static_cast(infile->Get("BinnedWorkspace")); @@ -174,7 +178,8 @@ static void BM_RooFit_BinnedTestHesse(benchmark::State &state) TFile *infile = new TFile("workspace.root"); // if (infile->IsZombie()) { buildBinnedTest(chan, nbins, 2, "workspace.root"); - if(verbose) std::cout << "Workspace for tests was created!" << std::endl; + if (verbose) + std::cout << "Workspace for tests was created!" << std::endl; // } infile = TFile::Open("workspace.root"); RooWorkspace *w = static_cast(infile->Get("BinnedWorkspace")); @@ -211,7 +216,8 @@ static void BM_RooFit_BinnedTestMinos(benchmark::State &state) auto infile = std::make_unique("workspace.root"); // if (infile->IsZombie()) { buildBinnedTest(chan, nbins, 2, "workspace.root"); - if(verbose) std::cout << "Workspace for tests was created!" << std::endl; + if (verbose) + std::cout << "Workspace for tests was created!" << std::endl; //} infile.reset(TFile::Open("workspace.root")); RooWorkspace *w = static_cast(infile->Get("BinnedWorkspace")); @@ -235,11 +241,11 @@ static void BM_RooFit_BinnedTestMinos(benchmark::State &state) } } -//############## Run # Tests ############################### +// ############## Run # Tests ############################### static void ChanArguments(benchmark::internal::Benchmark *b) { - // channel scan + // channel scan for (int nChannels : nChannelsVector) { for (int nCPU : nCPUVector) { b->Args({nChannels, nBinsForChannelScan, nCPU}); @@ -254,24 +260,26 @@ static void ChanArguments(benchmark::internal::Benchmark *b) } } -BENCHMARK(BM_RooFit_BinnedTestMigrad) - ->Apply(ChanArguments) - ->UseRealTime() - ->Unit(timeUnit) - ->Iterations(1); -BENCHMARK(BM_RooFit_BinnedTestHesse) - ->Apply(ChanArguments) - ->UseRealTime() - ->Unit(timeUnit) - ->Iterations(1); -BENCHMARK(BM_RooFit_BinnedTestMinos) - ->Apply(ChanArguments) - ->UseRealTime() - ->Unit(timeUnit) - ->Iterations(1); +BENCHMARK(BM_RooFit_BinnedTestMigrad)->Apply(ChanArguments)->UseRealTime()->Unit(timeUnit)->Iterations(1); +BENCHMARK(BM_RooFit_BinnedTestHesse)->Apply(ChanArguments)->UseRealTime()->Unit(timeUnit)->Iterations(1); +BENCHMARK(BM_RooFit_BinnedTestMinos)->Apply(ChanArguments)->UseRealTime()->Unit(timeUnit)->Iterations(1); + +// ############## End Of Tests ######################################## +// #################################################################### +// ############## RUN ################################################# + +int main(int argc, char **argv) +{ -//############## End Of Tests ######################################## -//#################################################################### -//############## RUN ################################################# + benchmark::Initialize(&argc, argv); -BENCHMARK_MAIN(); + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "-b") { + if (i + 1 < argc) { + // Set the evalBackend value from the next command-line argument + evalBackend = argv[i + 1]; + } + } + } + benchmark::RunSpecifiedBenchmarks(); +} diff --git a/root/roofit/roofit/compare_benchmarks.py b/root/roofit/roofit/compare_benchmarks.py new file mode 100644 index 000000000..d92734201 --- /dev/null +++ b/root/roofit/roofit/compare_benchmarks.py @@ -0,0 +1,52 @@ +import pandas as pd +import csv +import matplotlib.pyplot as plt +import numpy as np +from io import StringIO + + +def read_csv(csv_file_path): + with open(csv_file_path, "r") as csvfile: + lines = [line for line in csvfile if ("BM_RooFit" in line or "name," in line)] + print("\n".join(lines)) + return pd.read_csv(StringIO("\n".join(lines))) + + +codegen_df = read_csv("out_codegen.csv") +codegen_nograd_df = read_csv("out_codegen_ngrad.csv") +legacy_df = read_csv("out_legacy.csv") +cpu_df = read_csv("out_cpu.csv") + +# Plotting +plt.figure(figsize=(10, 6)) + +x = np.arange(len(codegen_df["name"].unique())) + + +for i, benchmark in enumerate(codegen_df["name"].unique()): + + codegen_time = codegen_df.loc[codegen_df["name"] == benchmark, "real_time"] + codegen_nograd_time = codegen_nograd_df.loc[codegen_nograd_df["name"] == benchmark, "real_time"] + cpu_time = cpu_df.loc[cpu_df["name"] == benchmark, "real_time"] + legacy_time = legacy_df.loc[legacy_df["name"] == benchmark, "real_time"] + + plt.bar(x[i] - 0.10, codegen_time, width=0.15, align="center", label="codegen", color="lightblue") + plt.bar(x[i], codegen_nograd_time, width=0.15, align="edge", label="codegen_nograd", color="navy") + plt.bar(x[i] + 0.15, cpu_time, width=0.15, align="edge", label="cpu", color="cyan") + plt.bar(x[i] + 0.30, legacy_time, width=0.15, align="edge", label="legacy", color="gray") + + +# Customize legend +legend_labels = ["codegen", "codegen_nograd", "cpu", "legacy"] +legend_colors = ["lightblue", "navy", "cyan", "gray"] +legend_handles = [plt.Rectangle((0, 0), 1, 1, color=color) for color in legend_colors] +plt.legend(legend_handles, legend_labels) + +plt.yscale("log") + +plt.xlabel("Benchmark") +plt.ylabel("Time (milliseconds)") +plt.title("Comparison of Benchmarks for Different Evaluation Backends") +plt.xticks(x, rotation=90) +plt.tight_layout() +plt.savefig("comparision_plot.png") diff --git a/root/roofit/roofit/run_benchmarks.sh b/root/roofit/roofit/run_benchmarks.sh new file mode 100755 index 000000000..308388f86 --- /dev/null +++ b/root/roofit/roofit/run_benchmarks.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Function to run a benchmark script and wait until the CSV file is generated +# to make it executable chmod +x run_benchmarks.sh +#then run it using ./run_benchmarks.sh +#!/bin/bash + +# Function to run the benchmark command and wait for CSV file to be generated +run_benchmark() { + echo "Running benchmark: $1" + $1 & + local pid=$! + while [ ! -f $2 ]; do + sleep 1 + done + wait $pid + echo "CSV file generated: $2" +} + +# Run benchmarks +run_benchmark "./benchRooFitBinned -b codegen --benchmark_out_format=csv --benchmark_out=out_codegen.csv" "out_codegen.csv" +run_benchmark "./benchRooFitBinned -b codegen_no_grad --benchmark_out_format=csv --benchmark_out=out_codegen_ngrad.csv" "out_codegen_ngrad.csv" +run_benchmark "./benchRooFitBinned -b legacy --benchmark_out_format=csv --benchmark_out=out_legacy.csv" "out_legacy.csv" +run_benchmark "./benchRooFitBinned -b cpu --benchmark_out_format=csv --benchmark_out=out_cpu.csv" "out_cpu.csv" + +# Run Python script +python3 compare_benchmarks.py \ No newline at end of file