From 43fbda86d7c5b5dfd9846470aefe2d7ca03464bb Mon Sep 17 00:00:00 2001
From: Charles Kawczynski <kawczynski.charles@gmail.com>
Date: Mon, 12 Sep 2022 11:07:52 -0700
Subject: [PATCH 1/3] Aim allocation monitoring at perf targets

---
 .buildkite/pipeline.yml        |  2 +-
 examples/hybrid/cli_options.jl | 10 ++++++++++
 perf/allocs.jl                 | 14 ++++++++++++--
 perf/allocs_per_case.jl        |  1 +
 4 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 66d52593159..7fc631f0c52 100755
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -369,7 +369,7 @@ steps:
         agents:
           slurm_mem: 20GB
 
-      - label: ":rocket: flame graph: threadedd perf target (ρe_tot)"
+      - label: ":rocket: flame graph: threaded perf target (ρe_tot)"
         command: "julia --threads 8 --color=yes --project=perf perf/flame.jl --job_id flame_perf_target_rhoe_threaded --enable_threading true"
         artifact_paths: "flame_perf_target_rhoe_threaded/*"
         agents:
diff --git a/examples/hybrid/cli_options.jl b/examples/hybrid/cli_options.jl
index 1d9d8e7e8fc..74eda0317e8 100644
--- a/examples/hybrid/cli_options.jl
+++ b/examples/hybrid/cli_options.jl
@@ -376,3 +376,13 @@ function parsed_args_per_job_id(buildkite_yaml; trigger = "driver.jl")
     end
     return result
 end
+
+function non_default_command_line_flags_parsed_args(parsed_args)
+    (s, default_parsed_args) = parse_commandline()
+    s = ""
+    for k in keys(parsed_args)
+        default_parsed_args[k] == parsed_args[k] && continue
+        s *= "--$k $(parsed_args[k]) "
+    end
+    return rstrip(s)
+end
diff --git a/perf/allocs.jl b/perf/allocs.jl
index b06635f1f88..c46c2db29b0 100644
--- a/perf/allocs.jl
+++ b/perf/allocs.jl
@@ -1,5 +1,6 @@
 # Track some important dependencies:
 example_dir = joinpath(dirname(@__DIR__), "examples")
+include(joinpath(example_dir, "hybrid", "cli_options.jl"));
 
 import ClimaCore
 import SciMLBase
@@ -24,9 +25,17 @@ dirs_to_monitor = String.(dirs_to_monitor)
 
 #! format: off
 
+dict = parsed_args_per_job_id(; trigger = "benchmark.jl")
+for k in keys(dict)
+    dict[k]["job_id"] = "allocs_"*dict[k]["job_id"]
+
+    # Lower resolution, since allocation tracking is expensive
+    dict[k]["h_elem"] = 6
+    dict[k]["z_elem"] = 15
+end
 cli_options = [
-    ("--job_id alloc_sphere_baroclinic_wave_rhoe"),
-    ("--vert_diff true --surface_scheme bulk --moist equil --forcing held_suarez --microphy 0M"),
+    non_default_command_line_flags_parsed_args(dict["perf_target_unthreaded"]),
+    non_default_command_line_flags_parsed_args(dict["perf_target_threaded"]),
 ]
 #! format: on
 
@@ -35,6 +44,7 @@ import ReportMetrics
 for clio in cli_options
     job_id = first(split(last(split(clio, "--job_id ")), " "))
     clio_in = split(clio, " ")
+    @info "CL options: `$clio_in`"
     ReportMetrics.report_allocs(;
         job_name = string(job_id),
         run_cmd = `$(Base.julia_cmd()) --project=perf/ --track-allocation=all perf/allocs_per_case.jl $clio_in`,
diff --git a/perf/allocs_per_case.jl b/perf/allocs_per_case.jl
index f2f9675de60..eff4b5bb703 100644
--- a/perf/allocs_per_case.jl
+++ b/perf/allocs_per_case.jl
@@ -15,5 +15,6 @@ catch err
 end
 
 OrdinaryDiffEq.step!(integrator) # compile first
+Profile.clear()
 Profile.clear_malloc_data()
 OrdinaryDiffEq.step!(integrator)

From 3df3cff0dcaad8fcf125710db33076c2b049e9c5 Mon Sep 17 00:00:00 2001
From: Charles Kawczynski <kawczynski.charles@gmail.com>
Date: Sun, 18 Sep 2022 11:51:46 -0700
Subject: [PATCH 2/3] Remove threaded allocation table

---
 perf/allocs.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/perf/allocs.jl b/perf/allocs.jl
index c46c2db29b0..8849381cc7d 100644
--- a/perf/allocs.jl
+++ b/perf/allocs.jl
@@ -35,7 +35,6 @@ for k in keys(dict)
 end
 cli_options = [
     non_default_command_line_flags_parsed_args(dict["perf_target_unthreaded"]),
-    non_default_command_line_flags_parsed_args(dict["perf_target_threaded"]),
 ]
 #! format: on
 

From 7155da15d71892627acd8fef8031a097d6e78af8 Mon Sep 17 00:00:00 2001
From: Charles Kawczynski <kawczynski.charles@gmail.com>
Date: Sun, 18 Sep 2022 14:49:10 -0700
Subject: [PATCH 3/3] Increase mem alloc limits

---
 perf/flame.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perf/flame.jl b/perf/flame.jl
index 106899da78c..92700323dc9 100644
--- a/perf/flame.jl
+++ b/perf/flame.jl
@@ -68,9 +68,9 @@ allocs = @allocated OrdinaryDiffEq.step!(integrator)
 
 allocs_limit = Dict()
 allocs_limit["flame_perf_target_rhoe"] = 10575616
-allocs_limit["flame_perf_target_rhoe_threaded"] = 12078128
+allocs_limit["flame_perf_target_rhoe_threaded"] = 12078608
 allocs_limit["flame_perf_target_rhoe_callbacks"] = 21740792
-allocs_limit["flame_perf_target_rhoe_ARS343"] = 24021744
+allocs_limit["flame_perf_target_rhoe_ARS343"] = 24056711
 
 if allocs < allocs_limit[job_id] * buffer
     @info "TODO: lower `allocs_limit[$job_id]` to: $(allocs)"