Merge #836

836: Aim allocation monitoring at perf targets r=charleskawczynski a=charleskawczynski This PR aims our allocation monitoring, at the performance targets, so that we get finer granularity of allocations across more functions. Co-authored-by: Charles Kawczynski <[email protected]>
CliMA · Sep 18, 2022 · 68b92f9 · 68b92f9
2 parents b3de7ce + 7155da1
commit 68b92f9
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 5 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -369,7 +369,7 @@ steps:
         agents:
           slurm_mem: 20GB
 
-      - label: ":rocket: flame graph: threadedd perf target (ρe_tot)"
+      - label: ":rocket: flame graph: threaded perf target (ρe_tot)"
         command: "julia --threads 8 --color=yes --project=perf perf/flame.jl --job_id flame_perf_target_rhoe_threaded --enable_threading true"
         artifact_paths: "flame_perf_target_rhoe_threaded/*"
         agents:

diff --git a/examples/hybrid/cli_options.jl b/examples/hybrid/cli_options.jl
@@ -376,3 +376,13 @@ function parsed_args_per_job_id(buildkite_yaml; trigger = "driver.jl")
     end
     return result
 end
+
+function non_default_command_line_flags_parsed_args(parsed_args)
+    (s, default_parsed_args) = parse_commandline()
+    s = ""
+    for k in keys(parsed_args)
+        default_parsed_args[k] == parsed_args[k] && continue
+        s *= "--$k $(parsed_args[k]) "
+    end
+    return rstrip(s)
+end
diff --git a/perf/allocs.jl b/perf/allocs.jl
@@ -1,5 +1,6 @@
 # Track some important dependencies:
 example_dir = joinpath(dirname(@__DIR__), "examples")
+include(joinpath(example_dir, "hybrid", "cli_options.jl"));
 
 import ClimaCore
 import SciMLBase
@@ -24,9 +25,16 @@ dirs_to_monitor = String.(dirs_to_monitor)
 
 #! format: off
 
+dict = parsed_args_per_job_id(; trigger = "benchmark.jl")
+for k in keys(dict)
+    dict[k]["job_id"] = "allocs_"*dict[k]["job_id"]
+
+    # Lower resolution, since allocation tracking is expensive
+    dict[k]["h_elem"] = 6
+    dict[k]["z_elem"] = 15
+end
 cli_options = [
-    ("--job_id alloc_sphere_baroclinic_wave_rhoe"),
-    ("--vert_diff true --surface_scheme bulk --moist equil --forcing held_suarez --microphy 0M"),
+    non_default_command_line_flags_parsed_args(dict["perf_target_unthreaded"]),
 ]
 #! format: on
 
@@ -35,6 +43,7 @@ import ReportMetrics
 for clio in cli_options
     job_id = first(split(last(split(clio, "--job_id ")), " "))
     clio_in = split(clio, " ")
+    @info "CL options: `$clio_in`"
     ReportMetrics.report_allocs(;
         job_name = string(job_id),
         run_cmd = `$(Base.julia_cmd()) --project=perf/ --track-allocation=all perf/allocs_per_case.jl $clio_in`,

diff --git a/perf/allocs_per_case.jl b/perf/allocs_per_case.jl
@@ -15,5 +15,6 @@ catch err
 end
 
 OrdinaryDiffEq.step!(integrator) # compile first
+Profile.clear()
 Profile.clear_malloc_data()
 OrdinaryDiffEq.step!(integrator)
diff --git a/perf/flame.jl b/perf/flame.jl
@@ -68,9 +68,9 @@ allocs = @allocated OrdinaryDiffEq.step!(integrator)
 
 allocs_limit = Dict()
 allocs_limit["flame_perf_target_rhoe"] = 10575616
-allocs_limit["flame_perf_target_rhoe_threaded"] = 12078128
+allocs_limit["flame_perf_target_rhoe_threaded"] = 12078608
 allocs_limit["flame_perf_target_rhoe_callbacks"] = 21740792
-allocs_limit["flame_perf_target_rhoe_ARS343"] = 24021744
+allocs_limit["flame_perf_target_rhoe_ARS343"] = 24056711
 
 if allocs < allocs_limit[job_id] * buffer
     @info "TODO: lower `allocs_limit[$job_id]` to: $(allocs)"