address more comments

.github/workflows/calibration_test.yaml

@@ -0,0 +1,22 @@
+name: ClimaCalibrate E2E Test
+on:
+  push:
+    tags: '*'
+  pull_request:
+  merge_group:
+
+jobs:
+  test:
+    name: ClimaCalibrate E2E Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: '1.10'
+      - run: |
+          julia --project=calibration/test -e 'using Pkg; Pkg.instantiate(;verbose=true)'
+          julia --project=calibration/test calibration/test/e2e_test.jl

calibration/experiments/sphere_held_suarez_rhoe_equilmoist/Manifest.toml

@@ -351,11 +351,9 @@ version = "0.24.0"
 
 [[deps.ClimaCalibrate]]
 deps = ["Distributions", "EnsembleKalmanProcesses", "JLD2", "Random", "TOML", "YAML"]
-git-tree-sha1 = "b281e4e1e6713cf5f26088837107842a64cbc987"
-repo-rev = "main"
-repo-url = "https://github.com/CliMA/ClimaCalibrate.jl.git"
+git-tree-sha1 = "39a5164998bb7b9f54c9504975551525ec0be2d5"
 uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
-version = "0.1.0"
+version = "0.0.1"
 
     [deps.ClimaCalibrate.extensions]
     CESExt = "CalibrateEmulateSample"

calibration/experiments/sphere_held_suarez_rhoe_equilmoist/model_config.yml

@@ -6,6 +6,9 @@ forcing: held_suarez
 precip_model: 0M
 job_id: sphere_held_suarez_rhoe_equilmoist
 output_dir: output/sphere_held_suarez_rhoe_equilmoist
+# This filepath is only valid on the Caltech central cluster, to reproduce this path run:
+# using ClimaUtilities.ClimaArtifacts; artifact = @clima_artifact("atmos_held_suarez_obs")
+# restart_file = joinpath(artifact, "day200.0.hdf5")
 restart_file: /groups/esm/ClimaArtifacts/artifacts/atmos_held_suarez_obs/day200.0.hdf5
 output_default_diagnostics: false
 diagnostics:

calibration/experiments/sphere_held_suarez_rhoe_equilmoist/postprocessing.jl

@@ -13,23 +13,9 @@ function convergence_plot(
     output = joinpath("output", "sphere_held_suarez_rhoe_equilmoist"),
 )
     u_vec = EKP.get_u(eki)
-    error_vec = Float64[]
-    spread_vec = Float64[]
-    for ensemble in u_vec
-        ensemble_error = 0.0
-        ensemble_spread = 0.0
-        ensemble_mean = mean(ensemble)
-        for i in ensemble
-            ensemble_error += abs(i - theta_star)^2
-            ensemble_spread += abs(i - ensemble_mean)^2
-        end
-        ensemble_error /= length(ensemble)
-        ensemble_spread /= length(ensemble)
-
-        push!(error_vec, ensemble_error)
-        push!(spread_vec, ensemble_spread)
-    end
-
+    meanabsdiff²(x, e) = sum(i -> abs(i - x)^2, e) / length(e)
+    error_vec = map(ensemble -> meanabsdiff²(theta_star, ensemble), u_vec)
+    error_vec = map(ensemble -> meanabsdiff²(mean(ensemble), ensemble), u_vec)
     phi_vec = transform_unconstrained_to_constrained(prior, u_vec)
     u_series = eachcol(reduce(vcat, u_vec))
     phi_series = eachcol(reduce(vcat, phi_vec))

calibration/model_interface.jl

@@ -11,17 +11,21 @@ import ClimaCalibrate:
 
 """
     set_up_forward_model(member, iteration, experiment_dir::AbstractString)
-    set_up_forward_model(member, iteration, ::ExperimentConfig)
+    set_up_forward_model(member, iteration, ::ExperimentConfig; experiment_dir)
     set_up_forward_model(member, iteration, config_dict::AbstractDict)
 
 Return an AtmosConfig object for the given member and iteration.
 
 Turns off default diagnostics and sets the TOML parameter file to the member's path.
 This assumes that the  config dictionary has an `output_dir` key.
 """
-function set_up_forward_model(member, iteration, ::ExperimentConfig)
+function set_up_forward_model(
+    member,
+    iteration,
+    ::ExperimentConfig;
+    experiment_dir = dirname(Base.active_project()),
+)
     # Assume experiment_dir is project dir
-    experiment_dir = dirname(Base.active_project())
     config_dict = YAML.load_file(joinpath(experiment_dir, "model_config.yml"))
     set_up_forward_model(member, iteration, config_dict::AbstractDict)
 end

calibration/test/Manifest.toml

@@ -311,13 +311,11 @@ version = "0.5.3"
 deps = ["Adapt", "ArgParse", "ArtifactWrappers", "Artifacts", "AtmosphericProfilesLibrary", "ClimaComms", "ClimaCore", "ClimaDiagnostics", "ClimaParams", "ClimaTimeSteppers", "ClimaUtilities", "CloudMicrophysics", "Colors", "Dates", "Dierckx", "DiffEqBase", "DocStringExtensions", "FastGaussQuadrature", "Insolation", "Interpolations", "IntervalSets", "Krylov", "LinearAlgebra", "Logging", "NCDatasets", "NVTX", "Pkg", "Printf", "RRTMGP", "Random", "RootSolvers", "SciMLBase", "StaticArrays", "Statistics", "StatsBase", "SurfaceFluxes", "Thermodynamics", "YAML"]
 path = "../.."
 uuid = "b2c96348-7fb7-4fe0-8da9-78d88439e717"
-version = "0.26.2"
+version = "0.26.3"
 
 [[deps.ClimaCalibrate]]
 deps = ["Distributions", "EnsembleKalmanProcesses", "JLD2", "Random", "TOML", "YAML"]
 git-tree-sha1 = "39a5164998bb7b9f54c9504975551525ec0be2d5"
-repo-rev = "main"
-repo-url = "https://github.com/CliMA/ClimaCalibrate.jl.git"
 uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
 version = "0.0.1"
 

calibration/test/e2e_test.jl

@@ -78,8 +78,8 @@ experiment_config = CAL.ExperimentConfig(;
 )
 
 # Minimal EKI test
-function minimal_eki_test(julia_eki)
-    params = EKP.get_u(julia_eki)
+function minimal_eki_test(eki)
+    params = EKP.get_u(eki)
     spread = map(x -> var(abs.(x)), params)
 
     # Spread should be heavily decreased as particles have converged
@@ -89,10 +89,12 @@ function minimal_eki_test(julia_eki)
 end
 
 # Caltech HPC backend
-backend = CAL.get_backend()
+if !(@isdefined backend)
+    backend = CAL.get_backend()
+end
 @info "Running Calibration E2E test on $backend"
-if backend <: CAL.SlurmBackend
-    slurm_kwargs = CAL.kwargs(time = 5)
+if backend() isa CAL.SlurmBackend
+    slurm_kwargs = CAL.kwargs(time = 10)
     test_eki = CAL.calibrate(
         backend,
         experiment_config;
@@ -109,7 +111,6 @@ end
 
 # Run calibration
 julia_eki = CAL.calibrate(CAL.JuliaBackend, experiment_config)
-minimal_eki_test(julia_eki)
 
 @testset "Julia-only comparison calibration" begin
     minimal_eki_test(julia_eki)
@@ -122,16 +123,16 @@ end
 end
 
 # Debug plots
-function scatter_plot(julia_eki::EKP.EnsembleKalmanProcess)
+function scatter_plot(eki::EKP.EnsembleKalmanProcess)
     f = CairoMakie.Figure(resolution = (800, 600))
     ax = CairoMakie.Axis(
         f[1, 1],
         ylabel = "Absolute Parameter Value",
-        xlabel = "TOA Radiative SW Flux 30day average",
+        xlabel = "Top of atmosphere radiative SW flux",
     )
 
-    g = vec.(EKP.get_g(julia_eki; return_array = true))
-    params = map(x -> abs.(x), vec.((EKP.get_u(julia_eki))))
+    g = vec.(EKP.get_g(eki; return_array = true))
+    params = map(x -> abs.(x), vec.((EKP.get_u(eki))))
 
     for (gg, uu) in zip(g, params)
         CairoMakie.scatter!(ax, gg, uu)
@@ -147,14 +148,14 @@ end
 
 scatter_plot(test_eki)
 
-function param_versus_iter_plot(julia_eki::EKP.EnsembleKalmanProcess)
+function param_versus_iter_plot(eki::EKP.EnsembleKalmanProcess)
     f = CairoMakie.Figure(resolution = (800, 600))
     ax = CairoMakie.Axis(
         f[1, 1],
         ylabel = "Parameter Value",
         xlabel = "Iteration",
     )
-    params = EKP.get_u(julia_eki)
+    params = EKP.get_u(eki)
     for (i, param) in enumerate(params)
         CairoMakie.scatter!(ax, fill(i, length(param)), vec(param))
     end

calibration/test/model_config.yml

@@ -2,7 +2,7 @@ dt: 2000secs
 t_end: 30days
 config: column
 h_elem: 1
-idealized_insolation: false
+insolation: timevarying
 job_id: calibration_end_to_end_test
 output_dir: calibration_end_to_end_test
 output_default_diagnostics: false