TRMM failing to initialize on GPU

[haakon-e]

The following code:

# on clima cluster:
#   srun -G1 -t 05:00:00 --pty bash -l
# in your ClimaAtmos.jl repo (main branch):
#   julia --project=examples
# then run this script:
import CUDA
# import AbbreviatedStackTraces

import ClimaComms as Comms
import ClimaAtmos as CA

main_context = Comms.SingletonCommsContext(Comms.CUDADevice())
Comms.init(main_context)
job_id = "les_trmm_box"
config_dict = CA.load_yaml_file("/home/haakon/ClimaAtmos.jl/config/model_configs/$job_id.yml")
config_dict["restart_file"] = "/home/haakon/trmm_test_restart/day0.0.hdf5"

config = CA.AtmosConfig(config_dict; job_id, comms_ctx = main_context)
simulation = CA.get_simulation(config);

yields the following GPU error

error message

[ Info: Allocating Y: 18.797 s (30220606 allocations: 1.93 GiB)
ERROR: GPU compilation of MethodInstance for ClimaCoreCUDAExt.knl_copyto!(::ClimaCore.DataLayouts.IJFH{…}, ::Base.Broadcast.Broadcasted{…}, ::ClimaCore.DataLayouts.UniversalSize{…}) failed
KernelError: passing and using non-bitstype argument

Argument 3 to your kernel function is of type Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.IJFHStyle{4, CUDA.CuArray{Float32, N, CUDA.DeviceMemory} where N}, NTuple{5, Base.OneTo{Int64}}, typeof(ClimaAtmos.SurfaceConditions.surface_state_to_conditions), Tuple{Tuple{ClimaAtmos.SurfaceConditions.var"#surface_state#18"{Float32, Float32, Float32, Float32, Float32}}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.LocalGeometry{(1, 2, 3), ClimaCore.Geometry.XYZPoint{Float32}, Float32, StaticArraysCore.SMatrix{3, 3, Float32, 9}}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, ClimaCore.DataLayouts.IJFH{Thermodynamics.PhaseEquil{Float32}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.IJFHStyle{4, CUDA.CuArray{Float32, N, CUDA.DeviceMemory} where N}, Nothing, typeof(ClimaAtmos.projected_vector_data), Tuple{CUDA.CuRefType{ClimaCore.Geometry.Contravariant1Vector}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.Covariant123Vector{Float32}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.LocalGeometry{(1, 2, 3), ClimaCore.Geometry.XYZPoint{Float32}, Float32, StaticArraysCore.SMatrix{3, 3, Float32, 9}}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}}}, Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.IJFHStyle{4, CUDA.CuArray{Float32, N, CUDA.DeviceMemory} where N}, Nothing, typeof(ClimaAtmos.projected_vector_data), Tuple{CUDA.CuRefType{ClimaCore.Geometry.Contravariant2Vector}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.Covariant123Vector{Float32}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.LocalGeometry{(1, 2, 3), ClimaCore.Geometry.XYZPoint{Float32}, Float32, StaticArraysCore.SMatrix{3, 3, Float32, 9}}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}}}, ClimaCore.DataLayouts.IJFH{Float32, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Tuple{Thermodynamics.Parameters.ThermodynamicsParameters{Float32}}, Tuple{SurfaceFluxes.Parameters.SurfaceFluxesParameters{Float32, SurfaceFluxes.UniversalFunctions.BusingerParams{Float32}, Thermodynamics.Parameters.ThermodynamicsParameters{Float32}}}, Tuple{ClimaAtmos.Parameters.SurfaceTemperatureParameters{Float32}}, Tuple{ClimaAtmos.AtmosModel{ClimaAtmos.BoxModel, ClimaAtmos.EquilMoistModel, ClimaAtmos.Microphysics1Moment, ClimaAtmos.QuadratureCloud, Nothing, Nothing, Nothing, ClimaAtmos.IdealizedOzone, ClimaAtmos.RadiationTRMM_LBA{AtmosphericProfilesLibrary.TimeZProfile{Interpolations.Extrapolation{Float32, 2, Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Interpolations.Flat{Nothing}}}}, Nothing, Nothing, Nothing, Bool, ClimaAtmos.UseAllTendency, ClimaAtmos.EDMFXModel{ClimaAtmos.NoEntrainment, ClimaAtmos.NoDetrainment, Val{false}, Val{false}, Val{false}, Val{false}}, Nothing, Nothing, Nothing, Nothing, Nothing, ClimaAtmos.Explicit, ClimaAtmos.Explicit, Nothing, ClimaAtmos.SmagorinskyLilly, Nothing, ClimaAtmos.ZonallySymmetricSST, ClimaAtmos.IdealizedInsolation, ClimaAtmos.PrescribedSurfaceTemperature, ClimaAtmos.ConstantAlbedo{Float32}, ClimaAtmos.AtmosNumerics{Val{:none}, Val{:none}, Val{:none}, Val{:none}, Nothing, Nothing}}}, CUDA.CuRefValue{Nothing}, Float32}}, which is not isbits:
  .args is of type Tuple{Tuple{ClimaAtmos.SurfaceConditions.var"#surface_state#18"{Float32, Float32, Float32, Float32, Float32}}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.LocalGeometry{(1, 2, 3), ClimaCore.Geometry.XYZPoint{Float32}, Float32, StaticArraysCore.SMatrix{3, 3, Float32, 9}}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, ClimaCore.DataLayouts.IJFH{Thermodynamics.PhaseEquil{Float32}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.IJFHStyle{4, CUDA.CuArray{Float32, N, CUDA.DeviceMemory} where N}, Nothing, typeof(ClimaAtmos.projected_vector_data), Tuple{CUDA.CuRefType{ClimaCore.Geometry.Contravariant1Vector}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.Covariant123Vector{Float32}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.LocalGeometry{(1, 2, 3), ClimaCore.Geometry.XYZPoint{Float32}, Float32, StaticArraysCore.SMatrix{3, 3, Float32, 9}}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}}}, Base.Broadcast.Broadcasted{ClimaCore.DataLayouts.IJFHStyle{4, CUDA.CuArray{Float32, N, CUDA.DeviceMemory} where N}, Nothing, typeof(ClimaAtmos.projected_vector_data), Tuple{CUDA.CuRefType{ClimaCore.Geometry.Contravariant2Vector}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.Covariant123Vector{Float32}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}, ClimaCore.DataLayouts.IJFH{ClimaCore.Geometry.LocalGeometry{(1, 2, 3), ClimaCore.Geometry.XYZPoint{Float32}, Float32, StaticArraysCore.SMatrix{3, 3, Float32, 9}}, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Vararg{Base.Slice{Base.OneTo{Int64}}, 4}}, true}}}}, ClimaCore.DataLayouts.IJFH{Float32, 4, SubArray{Float32, 4, CUDA.CuDeviceArray{Float32, 5, 1}, Tuple{Int64, Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Tuple{Thermodynamics.Parameters.ThermodynamicsParameters{Float32}}, Tuple{SurfaceFluxes.Parameters.SurfaceFluxesParameters{Float32, SurfaceFluxes.UniversalFunctions.BusingerParams{Float32}, Thermodynamics.Parameters.ThermodynamicsParameters{Float32}}}, Tuple{ClimaAtmos.Parameters.SurfaceTemperatureParameters{Float32}}, Tuple{ClimaAtmos.AtmosModel{ClimaAtmos.BoxModel, ClimaAtmos.EquilMoistModel, ClimaAtmos.Microphysics1Moment, ClimaAtmos.QuadratureCloud, Nothing, Nothing, Nothing, ClimaAtmos.IdealizedOzone, ClimaAtmos.RadiationTRMM_LBA{AtmosphericProfilesLibrary.TimeZProfile{Interpolations.Extrapolation{Float32, 2, Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Interpolations.Flat{Nothing}}}}, Nothing, Nothing, Nothing, Bool, ClimaAtmos.UseAllTendency, ClimaAtmos.EDMFXModel{ClimaAtmos.NoEntrainment, ClimaAtmos.NoDetrainment, Val{false}, Val{false}, Val{false}, Val{false}}, Nothing, Nothing, Nothing, Nothing, Nothing, ClimaAtmos.Explicit, ClimaAtmos.Explicit, Nothing, ClimaAtmos.SmagorinskyLilly, Nothing, ClimaAtmos.ZonallySymmetricSST, ClimaAtmos.IdealizedInsolation, ClimaAtmos.PrescribedSurfaceTemperature, ClimaAtmos.ConstantAlbedo{Float32}, ClimaAtmos.AtmosNumerics{Val{:none}, Val{:none}, Val{:none}, Val{:none}, Nothing, Nothing}}}, CUDA.CuRefValue{Nothing}, Float32} which is not isbits.
    .10 is of type Tuple{ClimaAtmos.AtmosModel{ClimaAtmos.BoxModel, ClimaAtmos.EquilMoistModel, ClimaAtmos.Microphysics1Moment, ClimaAtmos.QuadratureCloud, Nothing, Nothing, Nothing, ClimaAtmos.IdealizedOzone, ClimaAtmos.RadiationTRMM_LBA{AtmosphericProfilesLibrary.TimeZProfile{Interpolations.Extrapolation{Float32, 2, Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Interpolations.Flat{Nothing}}}}, Nothing, Nothing, Nothing, Bool, ClimaAtmos.UseAllTendency, ClimaAtmos.EDMFXModel{ClimaAtmos.NoEntrainment, ClimaAtmos.NoDetrainment, Val{false}, Val{false}, Val{false}, Val{false}}, Nothing, Nothing, Nothing, Nothing, Nothing, ClimaAtmos.Explicit, ClimaAtmos.Explicit, Nothing, ClimaAtmos.SmagorinskyLilly, Nothing, ClimaAtmos.ZonallySymmetricSST, ClimaAtmos.IdealizedInsolation, ClimaAtmos.PrescribedSurfaceTemperature, ClimaAtmos.ConstantAlbedo{Float32}, ClimaAtmos.AtmosNumerics{Val{:none}, Val{:none}, Val{:none}, Val{:none}, Nothing, Nothing}}} which is not isbits.
      .1 is of type ClimaAtmos.AtmosModel{ClimaAtmos.BoxModel, ClimaAtmos.EquilMoistModel, ClimaAtmos.Microphysics1Moment, ClimaAtmos.QuadratureCloud, Nothing, Nothing, Nothing, ClimaAtmos.IdealizedOzone, ClimaAtmos.RadiationTRMM_LBA{AtmosphericProfilesLibrary.TimeZProfile{Interpolations.Extrapolation{Float32, 2, Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Interpolations.Flat{Nothing}}}}, Nothing, Nothing, Nothing, Bool, ClimaAtmos.UseAllTendency, ClimaAtmos.EDMFXModel{ClimaAtmos.NoEntrainment, ClimaAtmos.NoDetrainment, Val{false}, Val{false}, Val{false}, Val{false}}, Nothing, Nothing, Nothing, Nothing, Nothing, ClimaAtmos.Explicit, ClimaAtmos.Explicit, Nothing, ClimaAtmos.SmagorinskyLilly, Nothing, ClimaAtmos.ZonallySymmetricSST, ClimaAtmos.IdealizedInsolation, ClimaAtmos.PrescribedSurfaceTemperature, ClimaAtmos.ConstantAlbedo{Float32}, ClimaAtmos.AtmosNumerics{Val{:none}, Val{:none}, Val{:none}, Val{:none}, Nothing, Nothing}} which is not isbits.
        .radiation_mode is of type ClimaAtmos.RadiationTRMM_LBA{AtmosphericProfilesLibrary.TimeZProfile{Interpolations.Extrapolation{Float32, 2, Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Interpolations.Flat{Nothing}}}} which is not isbits.
          .rad_profile is of type AtmosphericProfilesLibrary.TimeZProfile{Interpolations.Extrapolation{Float32, 2, Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Interpolations.Flat{Nothing}}} which is not isbits.
            .prof is of type Interpolations.Extrapolation{Float32, 2, Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Interpolations.Flat{Nothing}} which is not isbits.
              .itp is of type Interpolations.GriddedInterpolation{Float32, 2, Matrix{Float32}, Tuple{Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}, Interpolations.Gridded{Interpolations.Linear{Interpolations.Throw{Interpolations.OnGrid}}}}, Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}}} which is not isbits.
                .knots is of type Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, Vector{Float32}} which is not isbits.
                  .2 is of type Vector{Float32} which is not isbits.
                .coefs is of type Matrix{Float32} which is not isbits.


Stacktrace:
  [1] check_invocation(job::GPUCompiler.CompilerJob)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/2CW9L/src/validation.jl:92
  [2] macro expansion
    @ ~/.julia/packages/GPUCompiler/2CW9L/src/driver.jl:92 [inlined]
  [3] macro expansion
    @ ~/.julia/packages/TimerOutputs/6KVfH/src/TimerOutput.jl:253 [inlined]
  [4] 
    @ GPUCompiler ~/.julia/packages/GPUCompiler/2CW9L/src/driver.jl:90
  [5] codegen
    @ ~/.julia/packages/GPUCompiler/2CW9L/src/driver.jl:82 [inlined]
  [6] compile(target::Symbol, job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/2CW9L/src/driver.jl:79
  [7] compile
    @ ~/.julia/packages/GPUCompiler/2CW9L/src/driver.jl:74 [inlined]
  [8] #1145
    @ ~/.julia/packages/CUDA/2kjXI/src/compiler/compilation.jl:250 [inlined]
  [9] JuliaContext(f::CUDA.var"#1145#1148"{GPUCompiler.CompilerJob{…}}; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/2CW9L/src/driver.jl:34
 [10] JuliaContext(f::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/2CW9L/src/driver.jl:25
 [11] compile(job::GPUCompiler.CompilerJob)
    @ CUDA ~/.julia/packages/CUDA/2kjXI/src/compiler/compilation.jl:249
 [12] actual_compilation(cache::Dict{…}, src::Core.MethodInstance, world::UInt64, cfg::GPUCompiler.CompilerConfig{…}, compiler::typeof(CUDA.compile), linker::typeof(CUDA.link))
    @ GPUCompiler ~/.julia/packages/GPUCompiler/2CW9L/src/execution.jl:237
 [13] cached_compilation(cache::Dict{…}, src::Core.MethodInstance, cfg::GPUCompiler.CompilerConfig{…}, compiler::Function, linker::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/2CW9L/src/execution.jl:151
 [14] macro expansion
    @ ~/.julia/packages/CUDA/2kjXI/src/compiler/execution.jl:380 [inlined]
 [15] macro expansion
    @ ./lock.jl:267 [inlined]
 [16] cufunction(f::typeof(ClimaCoreCUDAExt.knl_copyto!), tt::Type{Tuple{…}}; kwargs::@Kwargs{always_inline::Bool})
    @ CUDA ~/.julia/packages/CUDA/2kjXI/src/compiler/execution.jl:375
 [17] cufunction
    @ ~/.julia/packages/CUDA/2kjXI/src/compiler/execution.jl:372 [inlined]
 [18] macro expansion
    @ ~/.julia/packages/CUDA/2kjXI/src/compiler/execution.jl:112 [inlined]
 [19] threads_via_occupancy(f!::typeof(ClimaCoreCUDAExt.knl_copyto!), args::Tuple{…})
    @ ClimaCoreCUDAExt ~/.julia/packages/ClimaCore/rSpyb/ext/cuda/cuda_utils.jl:94
 [20] copyto!(dest::ClimaCore.DataLayouts.IJFH{…}, bc::Base.Broadcast.Broadcasted{…}, ::ClimaCore.DataLayouts.ToCUDA)
    @ ClimaCoreCUDAExt ~/.julia/packages/ClimaCore/rSpyb/ext/cuda/data_layouts_copyto.jl:66
 [21] copyto!
    @ ~/.julia/packages/ClimaCore/rSpyb/src/DataLayouts/copyto.jl:34 [inlined]
 [22] materialize!
    @ ./broadcast.jl:914 [inlined]
 [23] materialize!
    @ ./broadcast.jl:911 [inlined]
 [24] update_surface_conditions!(Y::ClimaCore.Fields.FieldVector{…}, p::@NamedTuple{…}, t::Float32)
    @ ClimaAtmos.SurfaceConditions ~/ClimaAtmos.jl/src/surface_conditions/surface_conditions.jl:32
 [25] macro expansion
    @ ~/ClimaAtmos.jl/src/cache/precomputed_quantities.jl:510 [inlined]
 [26] set_precomputed_quantities!(Y::ClimaCore.Fields.FieldVector{…}, p::@NamedTuple{…}, t::Float32)
    @ ClimaAtmos ~/.julia/packages/NVTX/epWSw/src/macro.jl:194
 [27] build_cache(Y::ClimaCore.Fields.FieldVector{…}, atmos::ClimaAtmos.AtmosModel{…}, params::ClimaAtmos.Parameters.ClimaAtmosParameters{…}, surface_setup::ClimaAtmos.SurfaceConditions.TRMM_LBA, sim_info::@NamedTuple{…}, aerosol_names::Tuple{})
    @ ClimaAtmos ~/ClimaAtmos.jl/src/cache/cache.jl:157
 [28] macro expansion
    @ ~/ClimaAtmos.jl/src/solver/type_getters.jl:675 [inlined]
 [29] macro expansion
    @ ./timing.jl:503 [inlined]
 [30] macro expansion
    @ ~/ClimaAtmos.jl/src/utils/utilities.jl:323 [inlined]
 [31] get_simulation(config::ClimaAtmos.AtmosConfig{…})
    @ ClimaAtmos ~/ClimaAtmos.jl/src/solver/type_getters.jl:674
 [32] top-level scope
    @ REPL[11]:1
 [33] top-level scope
    @ none:1
Some type information was truncated. Use `show(err)` to see complete trace.

which appears to point towards this line of code:

ClimaAtmos.jl/src/surface_conditions/surface_conditions.jl

Lines 32 to 45 in 898cf2e

	@. sfc_conditions_values = surface_state_to_conditions(
	wrapped_sfc_setup,
	sfc_local_geometry_values,
	int_ts_values,
	projected_vector_data(CT1, int_u_values, int_local_geometry_values),
	projected_vector_data(CT2, int_u_values, int_local_geometry_values),
	int_z_values,
	thermo_params,
	surface_fluxes_params,
	surface_temp_params,
	atmos,
	sfc_temp_var,
	t,
	)

---

versioninfo

julia> versioninfo()
Julia Version 1.10.5
Commit 6f3fdf7b362 (2024-08-27 14:19 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 128 × AMD EPYC 7513 32-Core Processor
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, znver3)
Threads: 1 default, 0 interactive, 1 GC (on 128 virtual cores)
Environment:
  LD_LIBRARY_PATH = /usr/local/cuda/lib64:/usr/local/cuda/lib64:/clima/software/openmpi/4.1.5/lib:/usr/local/cuda/lib64
  LD_RUN_PATH_modshare = /clima/software/openmpi/4.1.5/lib:1
  JULIA_CUDA_MEMORY_POOL = none
  JULIA_LOAD_PATH = :/clima/software/openmpi/4.1.5:/clima/modulefiles/cuda
  LD_LIBRARY_PATH_modshare = /clima/software/openmpi/4.1.5/lib:1:/usr/local/cuda/lib64:1
  JULIA_MPI_HAS_CUDA = true
  JULIA_LOAD_PATH_modshare = :1:/clima/modulefiles/cuda:1:/clima/software/openmpi/4.1.5:1
  LD_RUN_PATH = /clima/software/openmpi/4.1.5/lib
  JULIA_NUM_THREADS = 1
  JULIA_EDITOR = code

contents of yml file:

# TRMM config
reference_job_id: "les_box"  # for plotting
initial_condition: TRMM_LBA
surface_setup: TRMM_LBA
rad: TRMM_LBA
config: "box"
# microphysics
moist: "equil"
precip_model: "1M"
# diffusion
implicit_diffusion: false
approximate_linear_solve_iters: 2
hyperdiff: "false" 
apply_limiter: false
smagorinsky_lilly: true
# time- and spatial discretization
x_elem: 10
x_max: 100e3  # 100 km
y_elem: 10
y_max: 100e3  # 100 km 
z_elem: 17
z_max: 17e3
z_stretch: false
ode_algo: "SSPKnoth"
dt: "1secs" 
t_end: "2mins" 
dt_cloud_fraction: "10mins"
restart_file: "/groups/esm/hervik/climaatmos-les-artifacts/les_isdac_day0.0.hdf5"
# diagnostics
dt_save_state_to_disk: "1mins"
# netcdf_interpolation_num_points: [30, 30, 150]
diagnostics:
  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, husra, hussn, cl, clw, cli, hussfc, evspsbl, hfes, pr]
    period: 10mins