#1992 broke the gpu multiple field solve for the land

[charleskawczynski]

xref:

#1992 removed the uniform_vertical_levels function. This led to a GPU error downstream in ClimaLand (see error message below). This PR reverts that change and makes a patch release - this has been tested from ClimaLand (see run here).

The error came up in ClimaLand in a PR switching the canopy temperature variable from being stepped explicitly to implicitly. We also step soil water content implicitly, but there has been no error with that case. In the soil-only case, our implicit variable is defined on a ExtrudedFiniteDifferenceSpace. In the new combined soil/canopy case, we have one implicit variable on a ExtrudedFiniteDifferenceSpace, and the other on a SpectralElementSpace2D. Maybe the bug comes from having two variables on two different spaces, or from the SpectralElementSpace2D specifically?

Here is the error (e.g. from this failing run):

ERROR: LoadError: InvalidIRError: compiling MethodInstance for ClimaCoreCUDAExt.multiple_field_solve_kernel!(::ClimaComms.CUDADevice, ::Tuple{…}, ::Tuple{…}, ::Tuple{…}, ::Tuple{…}, ::ClimaCore.Fields.Field{…}, ::ClimaCore.DataLayouts.UniversalSize{…}, ::Val{…}) resulted in invalid LLVM IR
Reason: unsupported dynamic function invocation (call to _single_field_solve!)
Stacktrace:
 [1] macro expansion
   @ ~/.julia/packages/ClimaCore/8E0O8/ext/cuda/matrix_fields_multiple_field_solve.jl:70
 [2] generated_single_field_solve!
   @ ~/.julia/packages/ClimaCore/8E0O8/ext/cuda/matrix_fields_multiple_field_solve.jl:56
 [3] multiple_field_solve_kernel!
   @ ~/.julia/packages/ClimaCore/8E0O8/ext/cuda/matrix_fields_multiple_field_solve.jl:95
Hint: catch this exception as `err` and call `code_typed(err; interactive = true)` to introspect the erronous code with Cthulhu.jl
Stacktrace:
  [1] check_ir(job::GPUCompiler.CompilerJob{GPUCompiler.PTXCompilerTarget, CUDA.CUDACompilerParams}, args::LLVM.Module)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/validation.jl:147
  [2] macro expansion
    @ ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:458 [inlined]
  [3] macro expansion
    @ ~/.julia/packages/TimerOutputs/Lw5SP/src/TimerOutput.jl:253 [inlined]
  [4] macro expansion
    @ ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:457 [inlined]
  [5] emit_llvm(job::GPUCompiler.CompilerJob; libraries::Bool, toplevel::Bool, optimize::Bool, cleanup::Bool, only_entry::Bool, validate::Bool)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/utils.jl:103
  [6] emit_llvm
    @ ~/.julia/packages/GPUCompiler/Y4hSX/src/utils.jl:97 [inlined]
  [7] codegen(output::Symbol, job::GPUCompiler.CompilerJob; libraries::Bool, toplevel::Bool, optimize::Bool, cleanup::Bool, strip::Bool, validate::Bool, only_entry::Bool, parent_job::Nothing)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:136
  [8] codegen
    @ ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:115 [inlined]
  [9] compile(target::Symbol, job::GPUCompiler.CompilerJob; libraries::Bool, toplevel::Bool, optimize::Bool, cleanup::Bool, strip::Bool, validate::Bool, only_entry::Bool)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:111
 [10] compile
    @ ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:103 [inlined]
 [11] #1145
    @ ~/.julia/packages/CUDA/Tl08O/src/compiler/compilation.jl:254 [inlined]
 [12] JuliaContext(f::CUDA.var"#1145#1148"{GPUCompiler.CompilerJob{GPUCompiler.PTXCompilerTarget, CUDA.CUDACompilerParams}}; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:52
 [13] JuliaContext(f::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/driver.jl:42
 [14] compile(job::GPUCompiler.CompilerJob)
    @ CUDA ~/.julia/packages/CUDA/Tl08O/src/compiler/compilation.jl:253
 [15] actual_compilation(cache::Dict{Any, CUDA.CuFunction}, src::Core.MethodInstance, world::UInt64, cfg::GPUCompiler.CompilerConfig{GPUCompiler.PTXCompilerTarget, CUDA.CUDACompilerParams}, compiler::typeof(CUDA.compile), linker::typeof(CUDA.link))
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/execution.jl:237
 [16] cached_compilation(cache::Dict{Any, CUDA.CuFunction}, src::Core.MethodInstance, cfg::GPUCompiler.CompilerConfig{GPUCompiler.PTXCompilerTarget, CUDA.CUDACompilerParams}, compiler::Function, linker::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/Y4hSX/src/execution.jl:151
 [17] macro expansion
    @ ~/.julia/packages/CUDA/Tl08O/src/compiler/execution.jl:369 [inlined]
 [18] macro expansion
    @ ./lock.jl:267 [inlined]
 [19] cufunction(f::typeof(ClimaCoreCUDAExt.multiple_field_solve_kernel!), tt::Type{Tuple{…}}; kwargs::@Kwargs{always_inline::Bool})
    @ CUDA ~/.julia/packages/CUDA/Tl08O/src/compiler/execution.jl:364
 [20] cufunction
    @ ~/.julia/packages/CUDA/Tl08O/src/compiler/execution.jl:361 [inlined]
 [21] macro expansion
    @ ~/.julia/packages/CUDA/Tl08O/src/compiler/execution.jl:112 [inlined]
 [22] threads_via_occupancy(f!::typeof(ClimaCoreCUDAExt.multiple_field_solve_kernel!), args::Tuple{ClimaComms.CUDADevice, Tuple{…}, Tuple{…}, Tuple{…}, Tuple{…}, ClimaCore.Fields.Field{…}, ClimaCore.DataLayouts.UniversalSize{…}, Val{…}})
    @ ClimaCoreCUDAExt ~/.julia/packages/ClimaCore/8E0O8/ext/cuda/cuda_utils.jl:94
 [23] macro expansion
    @ ~/.julia/packages/ClimaCore/8E0O8/ext/cuda/matrix_fields_multiple_field_solve.jl:40 [inlined]
 [24] multiple_field_solve!(::ClimaComms.CUDADevice, cache::ClimaCore.MatrixFields.FieldVectorView{…}, x::ClimaCore.MatrixFields.FieldVectorView{…}, A::ClimaCore.MatrixFields.FieldMatrix{…}, b::ClimaCore.MatrixFields.FieldVectorView{…}, x1::ClimaCore.Fields.Field{…})
    @ ClimaCoreCUDAExt ~/.julia/packages/NVTX/pfSOQ/src/macro.jl:194
 [25] multiple_field_solve!
    @ ~/.julia/packages/ClimaCore/8E0O8/src/MatrixFields/multiple_field_solver.jl:8 [inlined]
 [26] macro expansion
    @ ~/.julia/packages/ClimaCore/8E0O8/src/MatrixFields/field_matrix_solver.jl:276 [inlined]
 [27] run_field_matrix_solver!(::ClimaCore.MatrixFields.BlockDiagonalSolve, cache::ClimaCore.MatrixFields.FieldVectorView{…}, x::ClimaCore.MatrixFields.FieldVectorView{…}, A::ClimaCore.MatrixFields.FieldMatrix{…}, b::ClimaCore.MatrixFields.FieldVectorView{…})
    @ ClimaCore.MatrixFields ~/.julia/packages/NVTX/pfSOQ/src/macro.jl:194
 [28] macro expansion
    @ ~/.julia/packages/ClimaCore/8E0O8/src/MatrixFields/field_matrix_solver.jl:89 [inlined]
 [29] field_matrix_solve!(solver::ClimaCore.MatrixFields.FieldMatrixSolver{…}, x::ClimaCore.Fields.FieldVector{…}, A::ClimaCore.MatrixFields.FieldMatrix{…}, b::ClimaCore.Fields.FieldVector{…})
    @ ClimaCore.MatrixFields ~/.julia/packages/NVTX/pfSOQ/src/macro.jl:194
 [30] ldiv!
    @ ~/ClimaLand.jl/src/shared_utilities/implicit_timestepping.jl:177 [inlined]
 [31] macro expansion
    @ ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/nl_solvers/newtons_method.jl:605 [inlined]
 [32] solve_newton!(alg::ClimaTimeSteppers.NewtonsMethod{…}, cache::@NamedTuple{…}, x::ClimaCore.Fields.FieldVector{…}, f!::ClimaTimeSteppers.var"#99#103"{…}, j!::ClimaTimeSteppers.var"#100#104"{…}, post_implicit!::ClimaTimeSteppers.var"#101#105"{…}, post_implicit_last!::ClimaTimeSteppers.var"#102#106"{…})
    @ ClimaTimeSteppers ~/.julia/packages/NVTX/pfSOQ/src/macro.jl:194
 [33] update_stage!
    @ ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/solvers/imex_ark.jl:148 [inlined]
 [34] update_stage!
    @ ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/solvers/imex_ark.jl:93 [inlined]
 [35] update_stage!
    @ ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/solvers/imex_ark.jl:96 [inlined]
 [36] step_u!(integrator::ClimaTimeSteppers.DistributedODEIntegrator{…}, cache::ClimaTimeSteppers.IMEXARKCache{…})
    @ ClimaTimeSteppers ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/solvers/imex_ark.jl:71
 [37] macro expansion
    @ ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/integrators.jl:241 [inlined]
 [38] step_u!(integrator::ClimaTimeSteppers.DistributedODEIntegrator{…})
    @ ClimaTimeSteppers ~/.julia/packages/NVTX/pfSOQ/src/macro.jl:194
 [39] __step!(integrator::ClimaTimeSteppers.DistributedODEIntegrator{…})
    @ ClimaTimeSteppers ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/integrators.jl:213
 [40] macro expansion
    @ ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/integrators.jl:169 [inlined]
 [41] solve!(integrator::ClimaTimeSteppers.DistributedODEIntegrator{…})
    @ ClimaTimeSteppers ~/.julia/packages/NVTX/pfSOQ/src/macro.jl:194
 [42] __solve(::SciMLBase.ODEProblem{…}, ::ClimaTimeSteppers.IMEXAlgorithm{…}; kwargs::@Kwargs{…})
    @ ClimaTimeSteppers ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/integrators.jl:163
 [43] __solve
    @ ~/.julia/packages/ClimaTimeSteppers/Wc2TE/src/integrators.jl:161 [inlined]
 [44] solve_call(_prob::SciMLBase.ODEProblem{…}, args::ClimaTimeSteppers.IMEXAlgorithm{…}; merge_callbacks::Bool, kwargshandle::Nothing, kwargs::@Kwargs{…})
    @ DiffEqBase ~/.julia/packages/DiffEqBase/c8MAQ/src/solve.jl:612
 [45] solve_call
    @ DiffEqBase ~/.julia/packages/DiffEqBase/c8MAQ/src/solve.jl:569 [inlined]
 [46] #solve_up#53
    @ DiffEqBase ~/.julia/packages/DiffEqBase/c8MAQ/src/solve.jl:1080 [inlined]
 [47] solve_up
    @ DiffEqBase ~/.julia/packages/DiffEqBase/c8MAQ/src/solve.jl:1066 [inlined]
 [48] #solve#51
    @ DiffEqBase ~/.julia/packages/DiffEqBase/c8MAQ/src/solve.jl:1003 [inlined]
 [49] setup_and_solve_problem(; greet::Bool)
    @ Main ~/ClimaLand.jl/experiments/long_runs/land_region.jl:659
 [50] top-level scope
    @ ~/ClimaLand.jl/experiments/long_runs/land_region.jl:661
 [51] include(fname::String)
    @ Base.MainInclude ./client.jl:489
in expression starting at /home/jsloan/ClimaLand.jl/experiments/long_runs/land_region.jl:661