From 5bf2208b168c604706c8675599e63f757c1fd14e Mon Sep 17 00:00:00 2001 From: Dennis Yatunin Date: Tue, 9 Jul 2024 01:14:17 -0700 Subject: [PATCH] Add debugging info --- .../box_analytic_cosine_mountain_test.yml | 2 +- .../plane_analytic_cosine_mountain_test.yml | 2 +- examples/hybrid/driver.jl | 9 +- src/ClimaAtmos.jl | 2 - src/analytic_solutions/analytic_solutions.jl | 3 - src/disable_topography_dss.jl | 168 ------------------ src/utils/common_spaces.jl | 8 + 7 files changed, 12 insertions(+), 182 deletions(-) delete mode 100644 src/disable_topography_dss.jl diff --git a/config/model_configs/box_analytic_cosine_mountain_test.yml b/config/model_configs/box_analytic_cosine_mountain_test.yml index cbccbe47d88..2c634847959 100644 --- a/config/model_configs/box_analytic_cosine_mountain_test.yml +++ b/config/model_configs/box_analytic_cosine_mountain_test.yml @@ -9,7 +9,7 @@ x_elem: 30 y_elem: 30 z_elem: 100 z_stretch: false -dt: "8secs" +dt: "6secs" t_end: "24hours" use_reference_state: false rayleigh_sponge: true diff --git a/config/model_configs/plane_analytic_cosine_mountain_test.yml b/config/model_configs/plane_analytic_cosine_mountain_test.yml index cdfa35f157d..308cd9fb054 100644 --- a/config/model_configs/plane_analytic_cosine_mountain_test.yml +++ b/config/model_configs/plane_analytic_cosine_mountain_test.yml @@ -7,7 +7,7 @@ z_max: 21e3 x_elem: 30 z_elem: 100 z_stretch: false -dt: "8secs" +dt: "6secs" t_end: "24hours" use_reference_state: false rayleigh_sponge: true diff --git a/examples/hybrid/driver.jl b/examples/hybrid/driver.jl index eee538127de..fbb915d53e5 100644 --- a/examples/hybrid/driver.jl +++ b/examples/hybrid/driver.jl @@ -10,13 +10,8 @@ import ClimaAtmos as CA import Random Random.seed!(1234) -# TODO: Fix the bug that's making this necessary for 2D topography runs. -import ClimaCore.DataLayouts: promote_parent_array_type -import CUDA -promote_parent_array_type( - ::Type{CUDA.CuArray{T1, N, B} where {N}}, - ::Type{Array{T2, N} where {N}}, -) where {T1, T2, B} = CUDA.CuArray{promote_type(T1, T2), N, B} where {N} +include("fix_1d_spectral_space_on_gpu.jl") +# include("disable_topography_dss.jl") if !(@isdefined config) (; config_file, job_id) = CA.commandline_kwargs() diff --git a/src/ClimaAtmos.jl b/src/ClimaAtmos.jl index a589dfc6644..d67f87a1e0e 100644 --- a/src/ClimaAtmos.jl +++ b/src/ClimaAtmos.jl @@ -1,7 +1,5 @@ module ClimaAtmos -# include("disable_topography_dss.jl") - using NVTX, Colors import Thermodynamics as TD diff --git a/src/analytic_solutions/analytic_solutions.jl b/src/analytic_solutions/analytic_solutions.jl index b88d309ea59..15eb0093bf1 100644 --- a/src/analytic_solutions/analytic_solutions.jl +++ b/src/analytic_solutions/analytic_solutions.jl @@ -43,9 +43,6 @@ function initial_thermo_state(params, coord) T_min = FT(100) z_iso = log((T_min / T_sfc - a) / (1 - a)) / β p_iso = p_sfc * ((1 - a) / (1 - a * T_sfc / T_min))^(cp_d / R_d) - z > z_iso && - @warn "Extending constant-N profile with isothermal profile above T = \ - $T_min (z = $z_iso)" maxlog = 1 # The perturbation in approximate_FΔuvw is computed around a background # state that is not hydrostatically balanced and is unphysical above 30 km, diff --git a/src/disable_topography_dss.jl b/src/disable_topography_dss.jl deleted file mode 100644 index a34b37cdbd7..00000000000 --- a/src/disable_topography_dss.jl +++ /dev/null @@ -1,168 +0,0 @@ -import ClimaCore: - DataLayouts, - Geometry, - Topologies, - Quadratures, - Grids, - Hypsography, - Spaces, - Fields, - Operators -import ClimaTimeSteppers -import ClimaComms -using StaticArrays: SVector, SMatrix -using LinearAlgebra: adjoint, Adjoint, ldiv!, DenseMatrix, lu, norm - -ClimaTimeSteppers.NVTX.@annotate function ClimaTimeSteppers.solve_newton!( - alg::ClimaTimeSteppers.NewtonsMethod, - cache, - x, - f!, - j! = nothing, - pre_iteration! = nothing, - post_solve! = nothing, -) - (; max_iters, update_j, krylov_method, convergence_checker, verbose) = alg - (; krylov_method_cache, convergence_checker_cache) = cache - (; Δx, f, j) = cache - if (!isnothing(j)) && ClimaTimeSteppers.needs_update!( - update_j, - ClimaTimeSteppers.NewNewtonSolve(), - ) - j!(j, x) - end - for n in 1:max_iters - # Compute Δx[n]. - if (!isnothing(j)) && ClimaTimeSteppers.needs_update!( - update_j, - ClimaTimeSteppers.NewNewtonIteration(), - ) - j!(j, x) - end - f!(f, x) - if isnothing(krylov_method) - if j isa DenseMatrix - ldiv!(Δx, lu(j), f) # Highly inefficient! Only used for testing. - else - ldiv!(Δx, j, f) - end - else - ClimaTimeSteppers.solve_krylov!( - krylov_method, - krylov_method_cache, - Δx, - x, - f!, - f, - n, - pre_iteration!, - j, - ) - end - ClimaTimeSteppers.is_verbose(verbose) && - @info "Newton iteration $n: ‖x‖ = $(norm(x)), ‖Δx‖ = $(norm(Δx))" - - c_dss_buffer = Spaces.create_dss_buffer(Δx.c) - f_dss_buffer = Spaces.create_dss_buffer(Δx.f) - Spaces.weighted_dss!(Δx.c => c_dss_buffer, Δx.f => f_dss_buffer) - - x .-= Δx - # Update x[n] with Δx[n - 1], and exit the loop if Δx[n] is not needed. - # Check for convergence if necessary. - if ClimaTimeSteppers.is_converged!( - convergence_checker, - convergence_checker_cache, - x, - Δx, - n, - ) - isnothing(post_solve!) || post_solve!(x) - break - elseif n == max_iters - isnothing(post_solve!) || post_solve!(x) - else - isnothing(pre_iteration!) || pre_iteration!(x) - end - if ClimaTimeSteppers.is_verbose(verbose) && n == max_iters - @warn "Newton's method did not converge within $n iterations: ‖x‖ = $(norm(x)), ‖Δx‖ = $(norm(Δx))" - end - end -end - -function Grids._ExtrudedFiniteDifferenceGrid( - horizontal_grid::Grids.AbstractGrid, - vertical_grid::Grids.FiniteDifferenceGrid, - adaption::Grids.HypsographyAdaption, - global_geometry::Geometry.AbstractGlobalGeometry, - face_z::DataLayouts.AbstractData{Geometry.ZPoint{FT}}, -) where {FT} - # construct the "flat" grid - # avoid cached constructor so that it gets cleaned up automatically - flat_grid = Grids._ExtrudedFiniteDifferenceGrid( - horizontal_grid, - vertical_grid, - Grids.Flat(), - global_geometry, - ) - center_flat_space = Spaces.space(flat_grid, Grids.CellCenter()) - face_flat_space = Spaces.space(flat_grid, Grids.CellFace()) - - # compute the "z-only local geometry" based on face z coords - ArrayType = ClimaComms.array_type(horizontal_grid.topology) - # currently only works on Arrays - (center_z_local_geometry, face_z_local_geometry) = Grids.fd_geometry_data( - Adapt.adapt(Array, face_z); - periodic = Topologies.isperiodic(vertical_grid.topology), - ) - - center_z_local_geometry = Adapt.adapt(ArrayType, center_z_local_geometry) - face_z_local_geometry = Adapt.adapt(ArrayType, face_z_local_geometry) - - # compute ∇Z at face and centers - grad = Operators.Gradient() - - center_∇Z_field = - grad.( - Fields.Field( - center_z_local_geometry, - center_flat_space, - ).coordinates.z - ) - face_∇Z_field = - grad.( - Fields.Field(face_z_local_geometry, face_flat_space).coordinates.z - ) - # buffer = (; - # c = Spaces.create_dss_buffer(center_∇Z_field), - # f = Spaces.create_dss_buffer(face_∇Z_field), - # ) - - # Spaces.weighted_dss!(center_∇Z_field => buffer.c, face_∇Z_field => buffer.f) - - # construct full local geometry - center_local_geometry = - Geometry.product_geometry.( - horizontal_grid.local_geometry, - center_z_local_geometry, - Ref(global_geometry), - Ref(Geometry.WVector(1)) .* - adjoint.(Fields.field_values(center_∇Z_field)), - ) - face_local_geometry = - Geometry.product_geometry.( - horizontal_grid.local_geometry, - face_z_local_geometry, - Ref(global_geometry), - Ref(Geometry.WVector(1)) .* - adjoint.(Fields.field_values(face_∇Z_field)), - ) - - return Grids.ExtrudedFiniteDifferenceGrid( - horizontal_grid, - vertical_grid, - adaption, - global_geometry, - center_local_geometry, - face_local_geometry, - ) -end diff --git a/src/utils/common_spaces.jl b/src/utils/common_spaces.jl index 7feb5f38059..f12e89dc3b4 100644 --- a/src/utils/common_spaces.jl +++ b/src/utils/common_spaces.jl @@ -120,6 +120,14 @@ function make_hybrid_spaces( hypsography = Hypsography.LinearAdaption(Geometry.ZPoint.(z_surface)) end + @show device + @show ClimaComms.device(Fields.coordinate_field(h_space)) + @show ClimaComms.device(z_grid) + @show ClimaComms.device(z_surface) + @show ClimaComms.device(Geometry.ZPoint.(z_surface)) + @show typeof(h_space) + @show typeof(z_grid) + @show typeof(z_surface) end grid = Grids.ExtrudedFiniteDifferenceGrid(h_grid, z_grid, hypsography; deep) # TODO: return the grid