From bddbfc21fe861971b1a9b660c319138f35c63e46 Mon Sep 17 00:00:00 2001
From: Michael Cassidy <mc.mathematics@gmail.com>
Date: Thu, 24 Oct 2024 00:26:17 +1100
Subject: [PATCH] initial sketch of benchmarking

---
 SignalAnalysisBenchmarks/Project.toml         |  4 ++
 SignalAnalysisBenchmarks/etc/params.json      |  0
 SignalAnalysisBenchmarks/makefile             |  2 +
 SignalAnalysisBenchmarks/run.sh               | 17 ++++++
 .../src/SignalAnalysisBenchmarks.jl           | 52 +++++++++++++++++++
 .../src/dsp/DspBenchmarks.jl                  | 26 ++++++++++
 6 files changed, 101 insertions(+)
 create mode 100644 SignalAnalysisBenchmarks/Project.toml
 create mode 100644 SignalAnalysisBenchmarks/etc/params.json
 create mode 100644 SignalAnalysisBenchmarks/makefile
 create mode 100755 SignalAnalysisBenchmarks/run.sh
 create mode 100644 SignalAnalysisBenchmarks/src/SignalAnalysisBenchmarks.jl
 create mode 100644 SignalAnalysisBenchmarks/src/dsp/DspBenchmarks.jl

diff --git a/SignalAnalysisBenchmarks/Project.toml b/SignalAnalysisBenchmarks/Project.toml
new file mode 100644
index 0000000..cbfbff4
--- /dev/null
+++ b/SignalAnalysisBenchmarks/Project.toml
@@ -0,0 +1,4 @@
+name='SignalAnalysisBenchmarks'
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+SignalAnalysis = "df1fea92-c066-49dd-8b36-eace3378ea47"
diff --git a/SignalAnalysisBenchmarks/etc/params.json b/SignalAnalysisBenchmarks/etc/params.json
new file mode 100644
index 0000000..e69de29
diff --git a/SignalAnalysisBenchmarks/makefile b/SignalAnalysisBenchmarks/makefile
new file mode 100644
index 0000000..13cf414
--- /dev/null
+++ b/SignalAnalysisBenchmarks/makefile
@@ -0,0 +1,2 @@
+test:
+	./run.sh
diff --git a/SignalAnalysisBenchmarks/run.sh b/SignalAnalysisBenchmarks/run.sh
new file mode 100755
index 0000000..7dce3c3
--- /dev/null
+++ b/SignalAnalysisBenchmarks/run.sh
@@ -0,0 +1,17 @@
+#!/bin/zsh
+MOD=SignalAnalysisBenchmarks
+
+DT=$(date '+%Y-%m-%d_%H:%M:%S')
+TMP=/tmp/SignalAnalysis
+[ -d $TMP ] || mkdir -p $TMP
+FNAME="$TMP/regressions_$DT.json"
+SCRIPT=$(cat <<EOF
+using $MOD 
+using BenchmarkTools
+$MOD.load!("dsp";tune=false)
+@info "running benchmark suite..."
+results = run($MOD.SUITE["dsp"])
+BenchmarkTools.save("$FNAME", results)
+EOF
+)
+julia --project=. -e $SCRIPT --color=no
diff --git a/SignalAnalysisBenchmarks/src/SignalAnalysisBenchmarks.jl b/SignalAnalysisBenchmarks/src/SignalAnalysisBenchmarks.jl
new file mode 100644
index 0000000..ab68242
--- /dev/null
+++ b/SignalAnalysisBenchmarks/src/SignalAnalysisBenchmarks.jl
@@ -0,0 +1,52 @@
+module SignalAnalysisBenchmarks
+using BenchmarkTools
+using SignalAnalysis
+
+BenchmarkTools.DEFAULT_PARAMETERS.seconds = 1.0
+BenchmarkTools.DEFAULT_PARAMETERS.samples = 10000
+BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = 0.15
+BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = 0.01
+
+const PARAMS_PATH = joinpath(dirname(@__FILE__), "..", "etc", "params.json")
+const SUITE = BenchmarkGroup()
+
+    const MODULES = Dict( 
+        "dsp" => :DspBenchmarks
+    )
+
+load!(id::AbstractString; kwargs...) = load!(SUITE, id; kwargs...)
+
+function load!(group::BenchmarkGroup, id::AbstractString; tune::Bool = true)
+    modsym = MODULES[id]
+    modpath = joinpath(dirname(@__FILE__), id, "$(modsym).jl")
+    Core.eval(SignalAnalysisBenchmarks, :(include($modpath)))
+    modsuite = Core.eval(SignalAnalysisBenchmarks, modsym).SUITE
+    group[id] = modsuite
+    if tune
+        results = BenchmarkTools.load(PARAMS_PATH)[1]
+        haskey(results, id) && loadparams!(modsuite, results[id], :evals)
+    end
+    return group
+end
+
+#TODO: add tuning
+function loadall!(group::BenchmarkGroup; verbose::Bool = true, tune::Bool = false)
+    for id in keys(MODULES)
+        if verbose
+            print("loading group $(repr(id))... ")
+            time = @elapsed load!(group, id, tune=false)
+            println("done (took $time seconds)")
+        else
+            load!(group, id, tune=false)
+        end
+    end
+    if tune
+        results = BenchmarkTools.load(PARAMS_PATH)[1]
+        for (id, suite) in group
+            haskey(results, id) && loadparams!(suite, results[id], :evals)
+        end
+    end
+    return group
+end
+
+end
\ No newline at end of file
diff --git a/SignalAnalysisBenchmarks/src/dsp/DspBenchmarks.jl b/SignalAnalysisBenchmarks/src/dsp/DspBenchmarks.jl
new file mode 100644
index 0000000..6505ec7
--- /dev/null
+++ b/SignalAnalysisBenchmarks/src/dsp/DspBenchmarks.jl
@@ -0,0 +1,26 @@
+module DspBenchmarks
+    using SignalAnalysis: goertzel
+    using BenchmarkTools
+    
+    const SUITE = BenchmarkGroup()
+    SUITE["goertzel"] = BenchmarkGroup(["math","dsp"])
+    # function goertzel(x::AbstractVector, f, n; fs=framerate(x))
+    #   function goertzel(x::AbstractVector, f; fs=framerate(x))
+    #   function goertzel(x::AbstractMatrix, f, n; fs=framerate(x))k
+    goertzel_containers = [Vector,Matrix];
+    datatypes = [Float64];
+    #can feed a vector with a block size or not
+    Fs = 34100;
+    dims = [(Fs* duration,n_sensors) for n_sensors in 2:2:10 for duration in 1:2:10];
+    vector_sizes = (1:2:10) .* Fs
+    ns = [13 14 15].^2;
+    Ts =[Float64,Float32,Float64];
+    for n in ns
+        for vector_size in vector_sizes
+            for T ∈ Ts 
+                V = rand(T,vector_size)
+                SUITE["goertzel_vector"][string(vector_size),string(eltype(T))] = @benchmarkable goertzel($V,2200,$n;fs = $Fs)
+            end
+        end 
+    end
+end
\ No newline at end of file