diff --git a/NEWS.md b/NEWS.md index b283331249..430b39fcdd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,11 @@ ClimaCore.jl Release Notes main ------- +v0.15.0 +------- + +- ![][badge-💥breaking] support for `IntervalTopology(::Mesh)` has been dropped in favor of using `IntervalTopology(::ClimaComms.AbstractDevice, ::Mesh)`. PR [#1821](https://github.com/CliMA/ClimaCore.jl/pull/1821). + v0.14.9 ------- diff --git a/docs/tutorials/introduction.jl b/docs/tutorials/introduction.jl index f2164fcfce..b419cd96a7 100644 --- a/docs/tutorials/introduction.jl +++ b/docs/tutorials/introduction.jl @@ -53,9 +53,9 @@ rectangle_mesh = ClimaCore.Meshes.RectilinearMesh(rectangle_domain, 16, 16) # # A _topology_ determines the ordering and connections between elements of a mesh # - At the moment, this is only required for 2D meshes - +device = ClimaComms.device() rectangle_topology = ClimaCore.Topologies.Topology2D( - ClimaComms.SingletonCommsContext(), + ClimaComms.SingletonCommsContext(device), rectangle_mesh, ) #---------------------------------------------------------------------------- @@ -71,7 +71,8 @@ rectangle_topology = ClimaCore.Topologies.Topology2D( # # You can construct either the center or face space from the mesh, then construct the opposite space from the original one (this is to avoid allocating additional memory). -column_center_space = ClimaCore.Spaces.CenterFiniteDifferenceSpace(column_mesh) +column_center_space = + ClimaCore.Spaces.CenterFiniteDifferenceSpace(device, column_mesh) ## construct the face space from the center one column_face_space = ClimaCore.Spaces.FaceFiniteDifferenceSpace(column_center_space) diff --git a/examples/column/advect.jl b/examples/column/advect.jl index ee8ab459f8..7853ab976d 100644 --- a/examples/column/advect.jl +++ b/examples/column/advect.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: Fields, Domains, @@ -26,8 +28,8 @@ domain = Domains.IntervalDomain( boundary_names = (:left, :right), ) mesh = Meshes.IntervalMesh(domain, nelems = n) - -cs = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) V = Geometry.WVector.(ones(FT, fs)) diff --git a/examples/column/advect_diffusion.jl b/examples/column/advect_diffusion.jl index 554157730a..5bee5e4467 100644 --- a/examples/column/advect_diffusion.jl +++ b/examples/column/advect_diffusion.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: Fields, Domains, @@ -32,8 +34,8 @@ domain = Domains.IntervalDomain( boundary_names = (:bottom, :top), ) mesh = Meshes.IntervalMesh(domain, nelems = n) - -cs = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) zc = Fields.coordinate_field(cs) zp = (z₀ + z₁ / n / 2):(z₁ / n):(z₁ - z₁ / n / 2) diff --git a/examples/column/bb_fct_advection.jl b/examples/column/bb_fct_advection.jl index bf547e632d..8c049679f6 100644 --- a/examples/column/bb_fct_advection.jl +++ b/examples/column/bb_fct_advection.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends using Test using LinearAlgebra using OrdinaryDiffEq: ODEProblem, solve @@ -86,10 +88,10 @@ domain = Domains.IntervalDomain( stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(FT(7.0))) plot_string = ["uniform", "stretched"] - +device = ClimaComms.device() for (i, stretch_fn) in enumerate(stretch_fns) mesh = Meshes.IntervalMesh(domain, stretch_fn; nelems = n) - cent_space = Spaces.CenterFiniteDifferenceSpace(mesh) + cent_space = Spaces.CenterFiniteDifferenceSpace(device, mesh) face_space = Spaces.FaceFiniteDifferenceSpace(cent_space) z = Fields.coordinate_field(cent_space).z O = ones(FT, face_space) diff --git a/examples/column/ekman.jl b/examples/column/ekman.jl index b71e4a5dff..37860cd63d 100644 --- a/examples/column/ekman.jl +++ b/examples/column/ekman.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends using LinearAlgebra import ClimaCore: Fields, @@ -41,8 +43,8 @@ domain = Domains.IntervalDomain( ) #mesh = Meshes.IntervalMesh(domain, Meshes.ExponentialStretching(7.5e3); nelems = 30) mesh = Meshes.IntervalMesh(domain; nelems = nelems) - -cspace = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cspace = Spaces.CenterFiniteDifferenceSpace(device, mesh) fspace = Spaces.FaceFiniteDifferenceSpace(cspace) diff --git a/examples/column/fct_advection.jl b/examples/column/fct_advection.jl index e514bb0c60..6134ce4d41 100644 --- a/examples/column/fct_advection.jl +++ b/examples/column/fct_advection.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends using Test using LinearAlgebra using OrdinaryDiffEq: ODEProblem, solve, SSPRK33 @@ -87,11 +89,11 @@ initial_mass = zeros(FT, length(stretch_fns)) mass = zeros(FT, length(stretch_fns)) rel_mass_err = zeros(FT, length(stretch_fns)) plot_string = ["uniform", "stretched"] - +device = ClimaComms.device() for (i, stretch_fn) in enumerate(stretch_fns) mesh = Meshes.IntervalMesh(domain, stretch_fn; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) zc = Fields.coordinate_field(cs) O = ones(FT, fs) diff --git a/examples/column/heat.jl b/examples/column/heat.jl index 35a5fb2a89..cabe614ac2 100644 --- a/examples/column/heat.jl +++ b/examples/column/heat.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: Fields, Domains, @@ -28,8 +30,8 @@ domain = Domains.IntervalDomain( boundary_names = (:bottom, :top), ) mesh = Meshes.IntervalMesh(domain, nelems = n) - -cs = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) T = Fields.zeros(FT, cs) # Solve Heat Equation: ∂_t T = α ∇²T diff --git a/examples/column/hydrostatic.jl b/examples/column/hydrostatic.jl index c1ef7009e0..c2dd942b2e 100644 --- a/examples/column/hydrostatic.jl +++ b/examples/column/hydrostatic.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: Fields, Domains, @@ -32,8 +34,8 @@ domain = Domains.IntervalDomain( ) #mesh = Meshes.IntervalMesh(domain, Meshes.ExponentialStretching(7.5e3); nelems = 30) mesh = Meshes.IntervalMesh(domain; nelems = 30) - -cspace = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cspace = Spaces.CenterFiniteDifferenceSpace(device, mesh) fspace = Spaces.FaceFiniteDifferenceSpace(cspace) # https://github.com/CliMA/Thermodynamics.jl/blob/main/src/TemperatureProfiles.jl#L115-L155 diff --git a/examples/column/hydrostatic_discrete.jl b/examples/column/hydrostatic_discrete.jl index 8dc07786cb..f543ae6d46 100644 --- a/examples/column/hydrostatic_discrete.jl +++ b/examples/column/hydrostatic_discrete.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: Fields, Domains, @@ -34,8 +36,8 @@ domain = Domains.IntervalDomain( ) #mesh = Meshes.IntervalMesh(domain, Meshes.ExponentialStretching(7.5e3); nelems = 30) mesh = Meshes.IntervalMesh(domain; nelems = n_vert) - -cspace = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cspace = Spaces.CenterFiniteDifferenceSpace(device, mesh) fspace = Spaces.FaceFiniteDifferenceSpace(cspace) # https://github.com/CliMA/Thermodynamics.jl/blob/main/src/TemperatureProfiles.jl#L115-L155 diff --git a/examples/column/hydrostatic_ekman.jl b/examples/column/hydrostatic_ekman.jl index 49bc673269..483f037dd8 100644 --- a/examples/column/hydrostatic_ekman.jl +++ b/examples/column/hydrostatic_ekman.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends using LinearAlgebra import ClimaCore: @@ -45,8 +47,8 @@ domain = Domains.IntervalDomain( ) #mesh = Meshes.IntervalMesh(domain, Meshes.ExponentialStretching(7.5e3); nelems = 30) mesh = Meshes.IntervalMesh(domain; nelems = nelems) - -cspace = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cspace = Spaces.CenterFiniteDifferenceSpace(device, mesh) fspace = Spaces.FaceFiniteDifferenceSpace(cspace) diff --git a/examples/column/step.jl b/examples/column/step.jl index 7ab6583b31..70755a3eaa 100644 --- a/examples/column/step.jl +++ b/examples/column/step.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends using LinearAlgebra import ClimaCore: Fields, @@ -28,8 +30,8 @@ end domain = Domains.IntervalDomain(a, b, boundary_names = (:left, :right)) mesh = Meshes.IntervalMesh(domain, nelems = n) - -cs = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) V = Geometry.WVector.(ones(FT, fs)) diff --git a/examples/column/wave.jl b/examples/column/wave.jl index d06319fe25..9573e1dba1 100644 --- a/examples/column/wave.jl +++ b/examples/column/wave.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore.Geometry, LinearAlgebra import ClimaCore: Fields, @@ -22,8 +24,8 @@ domain = Domains.IntervalDomain( boundary_names = (:left, :right), ) mesh = Meshes.IntervalMesh(domain; nelems = 30) - -cspace = Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +cspace = Spaces.CenterFiniteDifferenceSpace(device, mesh) fspace = Spaces.FaceFiniteDifferenceSpace(cspace) zc = Fields.coordinate_field(cspace) diff --git a/examples/column/zalesak_fct_advection.jl b/examples/column/zalesak_fct_advection.jl index 123c43a332..9dc13a86a7 100644 --- a/examples/column/zalesak_fct_advection.jl +++ b/examples/column/zalesak_fct_advection.jl @@ -1,3 +1,5 @@ +import ClimaComms +ClimaComms.@import_required_backends using Test using LinearAlgebra using OrdinaryDiffEq: ODEProblem, solve @@ -87,10 +89,10 @@ domain = Domains.IntervalDomain( stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(FT(7.0))) plot_string = ["uniform", "stretched"] - +device = ClimaComms.device() for (i, stretch_fn) in enumerate(stretch_fns) mesh = Meshes.IntervalMesh(domain, stretch_fn; nelems = n) - cent_space = Spaces.CenterFiniteDifferenceSpace(mesh) + cent_space = Spaces.CenterFiniteDifferenceSpace(device, mesh) face_space = Spaces.FaceFiniteDifferenceSpace(cent_space) z = Fields.coordinate_field(cent_space).z O = ones(FT, face_space) diff --git a/examples/common_spaces.jl b/examples/common_spaces.jl index c20786224b..971e79690f 100644 --- a/examples/common_spaces.jl +++ b/examples/common_spaces.jl @@ -38,7 +38,7 @@ function make_horizontal_space( ) quad = Quadratures.GLL{npoly + 1}() if mesh isa Meshes.AbstractMesh1D - topology = Topologies.IntervalTopology(mesh) + topology = Topologies.IntervalTopology(ClimaComms.device(context), mesh) space = Spaces.SpectralElementSpace1D(topology, quad) elseif mesh isa Meshes.AbstractMesh2D topology = Topologies.Topology2D(context, mesh) @@ -68,8 +68,9 @@ function make_hybrid_spaces(h_space, z_max, z_elem; z_stretch) Geometry.ZPoint(z_max); boundary_names = (:bottom, :top), ) + context = ClimaComms.context(h_space) z_mesh = Meshes.IntervalMesh(z_domain, z_stretch; nelems = z_elem) - z_topology = Topologies.IntervalTopology(z_mesh) + z_topology = Topologies.IntervalTopology(ClimaComms.device(context), z_mesh) z_space = Spaces.CenterFiniteDifferenceSpace(z_topology) center_space = Spaces.ExtrudedFiniteDifferenceSpace(h_space, z_space) face_space = Spaces.FaceExtrudedFiniteDifferenceSpace(center_space) diff --git a/examples/hybrid/box/bubble_3d_invariant_rhotheta.jl b/examples/hybrid/box/bubble_3d_invariant_rhotheta.jl index a5b3ce5d11..9d141c37f5 100644 --- a/examples/hybrid/box/bubble_3d_invariant_rhotheta.jl +++ b/examples/hybrid/box/bubble_3d_invariant_rhotheta.jl @@ -2,6 +2,9 @@ using Test using ClimaComms using LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: ClimaCore, slab, @@ -47,6 +50,7 @@ function hvspace_3D( horzmesh = Meshes.RectilinearMesh(horzdomain, xelem, yelem) horztopology = Topologies.Topology2D(context, horzmesh) #horztopology = Topologies.Topology2D(horzmesh) + device = ClimaComms.device(context) zdomain = Domains.IntervalDomain( Geometry.ZPoint{FT}(zlim[1]), @@ -54,7 +58,7 @@ function hvspace_3D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(zdomain, nelems = zelem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace2D(horztopology, quad) diff --git a/examples/hybrid/box/bubble_3d_rhotheta.jl b/examples/hybrid/box/bubble_3d_rhotheta.jl index df20085eee..ab32a7df03 100644 --- a/examples/hybrid/box/bubble_3d_rhotheta.jl +++ b/examples/hybrid/box/bubble_3d_rhotheta.jl @@ -2,6 +2,9 @@ using Test using ClimaComms using LinearAlgebra, StaticArrays +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: ClimaCore, slab, @@ -42,6 +45,7 @@ function hvspace_3D( Geometry.YPoint{FT}(ylim[2]), periodic = true, ) + device = ClimaComms.device(context) horzdomain = Domains.RectangleDomain(xdomain, ydomain) horzmesh = Meshes.RectilinearMesh(horzdomain, xelem, yelem) @@ -53,7 +57,7 @@ function hvspace_3D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(zdomain, nelems = zelem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace2D(horztopology, quad) diff --git a/examples/hybrid/hybrid3dcs_dss.jl b/examples/hybrid/hybrid3dcs_dss.jl index cc5d2a1245..6272d64038 100644 --- a/examples/hybrid/hybrid3dcs_dss.jl +++ b/examples/hybrid/hybrid3dcs_dss.jl @@ -5,6 +5,9 @@ using JLD2 include("../nvtx.jl") +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: Domains, Fields, diff --git a/examples/hybrid/plane/bubble_2d_invariant_rhoe.jl b/examples/hybrid/plane/bubble_2d_invariant_rhoe.jl index 030e6e3028..1842e4fe31 100644 --- a/examples/hybrid/plane/bubble_2d_invariant_rhoe.jl +++ b/examples/hybrid/plane/bubble_2d_invariant_rhoe.jl @@ -3,6 +3,9 @@ push!(LOAD_PATH, joinpath(@__DIR__, "..", "..")) using Test using StaticArrays, IntervalSets, LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: ClimaCore, slab, @@ -35,7 +38,9 @@ function hvspace_2D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + context = ClimaComms.context() + device = ClimaComms.device(context) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]), @@ -43,7 +48,7 @@ function hvspace_2D( periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = xelem) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/examples/hybrid/plane/density_current_2d_flux_form.jl b/examples/hybrid/plane/density_current_2d_flux_form.jl index 800a37821a..fed1d3257b 100644 --- a/examples/hybrid/plane/density_current_2d_flux_form.jl +++ b/examples/hybrid/plane/density_current_2d_flux_form.jl @@ -1,6 +1,9 @@ using Test using LinearAlgebra, StaticArrays +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: ClimaCore, slab, @@ -33,7 +36,9 @@ function hvspace_2D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = velem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + context = ClimaComms.context() + device = ClimaComms.device(context) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]), @@ -41,7 +46,7 @@ function hvspace_2D( periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain; nelems = helem) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/examples/hybrid/plane/density_current_2dinvariant_rhoe.jl b/examples/hybrid/plane/density_current_2dinvariant_rhoe.jl index 65c7533ac9..83b36c4066 100644 --- a/examples/hybrid/plane/density_current_2dinvariant_rhoe.jl +++ b/examples/hybrid/plane/density_current_2dinvariant_rhoe.jl @@ -3,6 +3,9 @@ push!(LOAD_PATH, joinpath(@__DIR__, "..", "..")) using Test using StaticArrays, IntervalSets, LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: ClimaCore, slab, @@ -34,7 +37,9 @@ function hvspace_2D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + context = ClimaComms.context() + device = ClimaComms.device(context) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]), @@ -42,7 +47,7 @@ function hvspace_2D( periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = xelem) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/examples/hybrid/plane/inertial_gravity_wave.jl b/examples/hybrid/plane/inertial_gravity_wave.jl index ffa6d489b2..79f6f9a31b 100644 --- a/examples/hybrid/plane/inertial_gravity_wave.jl +++ b/examples/hybrid/plane/inertial_gravity_wave.jl @@ -6,6 +6,8 @@ include(joinpath("examples", "hybrid", "driver.jl")) using Printf using ProgressLogging using ClimaCorePlots, Plots +import ClimaComms +ClimaComms.@import_required_backends # Reference paper: https://rmets.onlinelibrary.wiley.com/doi/pdf/10.1002/qj.2105 diff --git a/examples/hybrid/plane/isothermal_channel.jl b/examples/hybrid/plane/isothermal_channel.jl index d7d8106f2e..65b1f99230 100644 --- a/examples/hybrid/plane/isothermal_channel.jl +++ b/examples/hybrid/plane/isothermal_channel.jl @@ -1,6 +1,9 @@ using Test using StaticArrays, IntervalSets, LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: ClimaCore, slab, @@ -44,7 +47,9 @@ function hvspace_2D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vert_face_space = Spaces.FaceFiniteDifferenceSpace(vertmesh) + context = ClimaComms.context() + device = ClimaComms.device(context) + vert_face_space = Spaces.FaceFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]), @@ -52,7 +57,7 @@ function hvspace_2D( periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = xelem) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) z_surface = Geometry.ZPoint.(warp_fn.(Fields.coordinate_field(horzspace))) diff --git a/examples/hybrid/plane/topo_agnesi_nh.jl b/examples/hybrid/plane/topo_agnesi_nh.jl index 283469fc31..2480bb320f 100644 --- a/examples/hybrid/plane/topo_agnesi_nh.jl +++ b/examples/hybrid/plane/topo_agnesi_nh.jl @@ -1,6 +1,8 @@ using Test using StaticArrays, IntervalSets, LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: ClimaCore, slab, @@ -57,7 +59,9 @@ function hvspace_2D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vert_face_space = Spaces.FaceFiniteDifferenceSpace(vertmesh) + context = ClimaComms.context() + device = ClimaComms.device(context) + vert_face_space = Spaces.FaceFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]), @@ -65,7 +69,7 @@ function hvspace_2D( periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = xelem) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/examples/hybrid/plane/topo_schar_nh.jl b/examples/hybrid/plane/topo_schar_nh.jl index 5c0225f851..9a5e905dec 100644 --- a/examples/hybrid/plane/topo_schar_nh.jl +++ b/examples/hybrid/plane/topo_schar_nh.jl @@ -1,6 +1,9 @@ using Test using StaticArrays, IntervalSets, LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends + import ClimaCore: ClimaCore, slab, @@ -80,7 +83,9 @@ function hvspace_2D( ) #vertmesh = Meshes.IntervalMesh(vertdomain, GeneralizedExponentialStretching(500.0, 5000.0), nelems = zelem) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vert_face_space = Spaces.FaceFiniteDifferenceSpace(vertmesh) + context = ClimaComms.context() + device = ClimaComms.device(context) + vert_face_space = Spaces.FaceFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]), @@ -88,7 +93,7 @@ function hvspace_2D( periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = xelem) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/examples/hybrid/sphere/baroclinic_wave_utilities.jl b/examples/hybrid/sphere/baroclinic_wave_utilities.jl index 53c00fd462..2464b2707e 100644 --- a/examples/hybrid/sphere/baroclinic_wave_utilities.jl +++ b/examples/hybrid/sphere/baroclinic_wave_utilities.jl @@ -1,3 +1,6 @@ +import ClimaComms +ClimaComms.@import_required_backends + # Constants required by "staggered_nonhydrostatic_model.jl" # const FT = ? # specified in each test file const p_0 = FT(1.0e5) diff --git a/examples/hybrid/sphere/deformation_flow.jl b/examples/hybrid/sphere/deformation_flow.jl index a5436c9706..5e2999758d 100644 --- a/examples/hybrid/sphere/deformation_flow.jl +++ b/examples/hybrid/sphere/deformation_flow.jl @@ -1,4 +1,5 @@ -using ClimaComms +import ClimaComms +ClimaComms.@import_required_backends using OrdinaryDiffEq using Test using Statistics: mean @@ -203,7 +204,8 @@ function run_deformation_flow(use_limiter, fct_op, dt) boundary_names = (:bottom, :top), ) vert_mesh = Meshes.IntervalMesh(vert_domain, nelems = zelem) - vert_cent_space = Spaces.CenterFiniteDifferenceSpace(vert_mesh) + device = ClimaComms.device(context) + vert_cent_space = Spaces.CenterFiniteDifferenceSpace(device, vert_mesh) horz_domain = Domains.SphereDomain(R) horz_mesh = Meshes.EquiangularCubedSphere(horz_domain, helem) diff --git a/examples/hybrid/sphere/hadley_circulation.jl b/examples/hybrid/sphere/hadley_circulation.jl index 3eebc63124..b489a182b2 100644 --- a/examples/hybrid/sphere/hadley_circulation.jl +++ b/examples/hybrid/sphere/hadley_circulation.jl @@ -1,4 +1,5 @@ -using ClimaComms +import ClimaComms +ClimaComms.@import_required_backends using Test using LinearAlgebra @@ -67,7 +68,8 @@ function sphere_3D( Meshes.ExponentialStretching(FT(7e3)); nelems = zelem, ) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + device = ClimaComms.device(context) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.SphereDomain(R) horzmesh = Meshes.EquiangularCubedSphere(horzdomain, helem) diff --git a/examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl b/examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl index 9e36c3167f..ba618afdf7 100644 --- a/examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl +++ b/examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl @@ -1,4 +1,5 @@ -using ClimaComms +import ClimaComms +ClimaComms.@import_required_backends using Test using LinearAlgebra @@ -58,7 +59,8 @@ function sphere_3D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + device = ClimaComms.device(context) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.SphereDomain(R) horzmesh = Meshes.EquiangularCubedSphere(horzdomain, helem) diff --git a/examples/hybrid/sphere/solid_body_rotation_3d.jl b/examples/hybrid/sphere/solid_body_rotation_3d.jl index 5089373b68..2c9eae36ca 100644 --- a/examples/hybrid/sphere/solid_body_rotation_3d.jl +++ b/examples/hybrid/sphere/solid_body_rotation_3d.jl @@ -57,7 +57,8 @@ function sphere_3D( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + device = ClimaComms.device(context) + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.SphereDomain(R) horzmesh = Meshes.EquiangularCubedSphere(horzdomain, helem) diff --git a/lib/ClimaCoreMakie/test/runtests.jl b/lib/ClimaCoreMakie/test/runtests.jl index 2edffdc331..cbab5e7696 100644 --- a/lib/ClimaCoreMakie/test/runtests.jl +++ b/lib/ClimaCoreMakie/test/runtests.jl @@ -87,7 +87,7 @@ end periodic = true, ) mesh = Meshes.IntervalMesh(domain, nelems = 32) - topology = Topologies.IntervalTopology(mesh) + topology = Topologies.IntervalTopology(ClimaComms.device(context), mesh) quad = Quadratures.GLL{5}() horz_space = Spaces.SpectralElementSpace1D(topology, quad) horz_coords = Fields.coordinate_field(horz_space) diff --git a/lib/ClimaCorePlots/Project.toml b/lib/ClimaCorePlots/Project.toml index e29511b020..8964752309 100644 --- a/lib/ClimaCorePlots/Project.toml +++ b/lib/ClimaCorePlots/Project.toml @@ -1,17 +1,18 @@ -authors = ["CliMA Contributors "] name = "ClimaCorePlots" uuid = "cf7c7e5a-b407-4c48-9047-11a94a308626" +authors = ["CliMA Contributors "] version = "0.2.10" +[deps] +ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" +ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" +RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +TriplotBase = "981d1d27-644d-49a2-9326-4793e63143c3" + [compat] ClimaCore = "0.11, 0.12, 0.13, 0.14" RecipesBase = "1" StaticArrays = "1" TriplotBase = "0.1" julia = "1.7" - -[deps] -ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" -RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" -StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" -TriplotBase = "981d1d27-644d-49a2-9326-4793e63143c3" diff --git a/lib/ClimaCorePlots/src/ClimaCorePlots.jl b/lib/ClimaCorePlots/src/ClimaCorePlots.jl index 6d11455bb5..1fe2dffcdf 100644 --- a/lib/ClimaCorePlots/src/ClimaCorePlots.jl +++ b/lib/ClimaCorePlots/src/ClimaCorePlots.jl @@ -3,6 +3,7 @@ module ClimaCorePlots import RecipesBase import TriplotBase +import ClimaComms import ClimaCore: ClimaCore, DataLayouts, @@ -317,7 +318,11 @@ function _slice_along(field, coord) end # construct the slice field space - htopo_ortho = ClimaCore.Topologies.IntervalTopology(hmesh_ortho) + context = ClimaComms.context(field) + htopo_ortho = ClimaCore.Topologies.IntervalTopology( + ClimaComms.device(context), + hmesh_ortho, + ) hspace_ortho = ClimaCore.Spaces.SpectralElementSpace1D( htopo_ortho, ClimaCore.Spaces.quadrature_style(hspace), diff --git a/lib/ClimaCorePlots/test/runtests.jl b/lib/ClimaCorePlots/test/runtests.jl index 46f3b04e8b..96990a7155 100644 --- a/lib/ClimaCorePlots/test/runtests.jl +++ b/lib/ClimaCorePlots/test/runtests.jl @@ -16,7 +16,9 @@ OUTPUT_DIR = mkpath(get(ENV, "CI_OUTPUT_DIR", tempname())) boundary_names = (:left, :right), ) mesh = ClimaCore.Meshes.IntervalMesh(domain; nelems = 5) - grid_topology = ClimaCore.Topologies.IntervalTopology(mesh) + context = ClimaComms.context() + device = ClimaComms.device(context) + grid_topology = ClimaCore.Topologies.IntervalTopology(device, mesh) quad = ClimaCore.Quadratures.GLL{5}() space = ClimaCore.Spaces.SpectralElementSpace1D(grid_topology, quad) coords = ClimaCore.Fields.coordinate_field(space) @@ -112,7 +114,9 @@ end boundary_names = (:bottom, :top), ) vertmesh = ClimaCore.Meshes.IntervalMesh(vertdomain, nelems = velem) - vert_center_space = ClimaCore.Spaces.CenterFiniteDifferenceSpace(vertmesh) + device = ClimaComms.device() + vert_center_space = + ClimaCore.Spaces.CenterFiniteDifferenceSpace(device, vertmesh) hv_center_space = ClimaCore.Spaces.ExtrudedFiniteDifferenceSpace( horz_space, @@ -199,8 +203,11 @@ end ClimaCore.Geometry.ZPoint{FT}(1000); boundary_names = (:bottom, :top), ) + context = ClimaComms.context() + device = ClimaComms.device(context) vertmesh = ClimaCore.Meshes.IntervalMesh(vertdomain, nelems = velem) - vert_center_space = ClimaCore.Spaces.CenterFiniteDifferenceSpace(vertmesh) + vert_center_space = + ClimaCore.Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = ClimaCore.Domains.IntervalDomain( ClimaCore.Geometry.XPoint{FT}(-500) .. @@ -208,7 +215,7 @@ end periodic = true, ) horzmesh = ClimaCore.Meshes.IntervalMesh(horzdomain; nelems = helem) - horztopology = ClimaCore.Topologies.IntervalTopology(horzmesh) + horztopology = ClimaCore.Topologies.IntervalTopology(device, horzmesh) quad = ClimaCore.Quadratures.GLL{npoly + 1}() horzspace = ClimaCore.Spaces.SpectralElementSpace1D(horztopology, quad) @@ -249,7 +256,9 @@ end boundary_names = (:bottom, :top), ) vertmesh = ClimaCore.Meshes.IntervalMesh(vertdomain, nelems = velem) - vert_center_space = ClimaCore.Spaces.CenterFiniteDifferenceSpace(vertmesh) + device = ClimaComms.device() + vert_center_space = + ClimaCore.Spaces.CenterFiniteDifferenceSpace(device, vertmesh) xdomain = ClimaCore.Domains.IntervalDomain( ClimaCore.Geometry.XPoint{FT}(-500) .. diff --git a/lib/ClimaCoreTempestRemap/test/netcdf.jl b/lib/ClimaCoreTempestRemap/test/netcdf.jl index 75dd2d7e64..7f69cf302a 100644 --- a/lib/ClimaCoreTempestRemap/test/netcdf.jl +++ b/lib/ClimaCoreTempestRemap/test/netcdf.jl @@ -1,5 +1,6 @@ import ClimaCore using ClimaComms +ClimaComms.@import_required_backends using ClimaCore: Geometry, Meshes, @@ -13,6 +14,7 @@ using NCDatasets using TempestRemap_jll using Test using ClimaCoreTempestRemap +device = ClimaComms.device() OUTPUT_DIR = mkpath(get(ENV, "CI_OUTPUT_DIR", tempname())) @@ -130,7 +132,7 @@ end boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = nlevels) - vspace = Spaces.CenterFiniteDifferenceSpace(vmesh) + vspace = Spaces.CenterFiniteDifferenceSpace(device, vmesh) hvspace = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace) fhvspace = Spaces.FaceExtrudedFiniteDifferenceSpace(hvspace) @@ -225,7 +227,7 @@ end boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = nlevels) - vspace = Spaces.CenterFiniteDifferenceSpace(vmesh) + vspace = Spaces.CenterFiniteDifferenceSpace(device, vmesh) hvspace = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace) fhvspace = Spaces.FaceExtrudedFiniteDifferenceSpace(hvspace) @@ -375,7 +377,7 @@ end boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = nlevels) - vfspace = Spaces.FaceFiniteDifferenceSpace(vmesh) + vfspace = Spaces.FaceFiniteDifferenceSpace(device, vmesh) z_surface = Geometry.ZPoint.(test_warp.(Fields.coordinate_field(hspace))) fhvspace = Spaces.ExtrudedFiniteDifferenceSpace( hspace, @@ -499,7 +501,7 @@ end boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = nlevels) - vspace = Spaces.CenterFiniteDifferenceSpace(vmesh) + vspace = Spaces.CenterFiniteDifferenceSpace(device, vmesh) hvspace = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace) fhvspace = Spaces.FaceExtrudedFiniteDifferenceSpace(hvspace) diff --git a/lib/ClimaCoreVTK/test/runtests.jl b/lib/ClimaCoreVTK/test/runtests.jl index 770a058558..acd3ce07d2 100644 --- a/lib/ClimaCoreVTK/test/runtests.jl +++ b/lib/ClimaCoreVTK/test/runtests.jl @@ -170,7 +170,8 @@ end periodic = true, ) hmesh = Meshes.IntervalMesh(hdomain, nelems = 10) - htopology = Topologies.IntervalTopology(hmesh) + context = ClimaComms.context() + htopology = Topologies.IntervalTopology(context, hmesh) quad = Quadratures.GLL{4}() hspace = Spaces.SpectralElementSpace1D(htopology, quad) @@ -179,7 +180,7 @@ end boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = 20) - vtopology = Topologies.IntervalTopology(vmesh) + vtopology = Topologies.IntervalTopology(context, vmesh) vspace = Spaces.FaceFiniteDifferenceSpace(vtopology) fspace = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace) @@ -208,6 +209,7 @@ end ) hmesh = Meshes.RectilinearMesh(hdomain, 4, 4) + context = ClimaComms.SingletonCommsContext() htopology = Topologies.Topology2D(ClimaComms.SingletonCommsContext(), hmesh) quad = Quadratures.GLL{4}() hspace = Spaces.SpectralElementSpace2D(htopology, quad) @@ -217,7 +219,7 @@ end boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = 20) - vtopology = Topologies.IntervalTopology(vmesh) + vtopology = Topologies.IntervalTopology(context, vmesh) vspace = Spaces.FaceFiniteDifferenceSpace(vtopology) fspace = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace) @@ -241,7 +243,8 @@ end hdomain = Domains.SphereDomain(R) hmesh = Meshes.EquiangularCubedSphere(hdomain, 4) - htopology = Topologies.Topology2D(ClimaComms.SingletonCommsContext(), hmesh) + context = ClimaComms.SingletonCommsContext() + htopology = Topologies.Topology2D(context, hmesh) quad = Quadratures.GLL{5}() hspace = Spaces.SpectralElementSpace2D(htopology, quad) @@ -252,7 +255,7 @@ end boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = 20) - vtopology = Topologies.IntervalTopology(vmesh) + vtopology = Topologies.IntervalTopology(context, vmesh) vspace = Spaces.FaceFiniteDifferenceSpace(vtopology) fspace = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace) diff --git a/src/Grids/finitedifference.jl b/src/Grids/finitedifference.jl index fea82453e8..aa7f739551 100644 --- a/src/Grids/finitedifference.jl +++ b/src/Grids/finitedifference.jl @@ -20,7 +20,7 @@ abstract type AbstractFiniteDifferenceGrid <: AbstractGrid end """ FiniteDifferenceGrid(topology::Topologies.IntervalTopology) - FiniteDifferenceGrid(mesh::Meshes.IntervalMesh) + FiniteDifferenceGrid(device::ClimaComms.AbstractDevice, mesh::Meshes.IntervalMesh) Construct a `FiniteDifferenceGrid` from an `IntervalTopology` (or an `IntervalMesh`). @@ -158,8 +158,10 @@ function fd_geometry_data( end -FiniteDifferenceGrid(mesh::Meshes.IntervalMesh) = - FiniteDifferenceGrid(Topologies.IntervalTopology(mesh)) +FiniteDifferenceGrid( + device::ClimaComms.AbstractDevice, + mesh::Meshes.IntervalMesh, +) = FiniteDifferenceGrid(Topologies.IntervalTopology(device, mesh)) # accessors topology(grid::FiniteDifferenceGrid) = grid.topology diff --git a/src/InputOutput/readers.jl b/src/InputOutput/readers.jl index fdf6de3c1c..537ce1c6f5 100644 --- a/src/InputOutput/readers.jl +++ b/src/InputOutput/readers.jl @@ -267,9 +267,9 @@ function read_topology_new(reader::HDF5Reader, name::AbstractString) type = attrs(group)["type"] if type == "IntervalTopology" mesh = read_mesh(reader, attrs(group)["mesh"]) - context = - ClimaComms.SingletonCommsContext(ClimaComms.device(reader.context)) - return Topologies.IntervalTopology(context, mesh) + device = ClimaComms.device(reader.context) + context = ClimaComms.SingletonCommsContext(device) + return Topologies.IntervalTopology(device, mesh) elseif type == "Topology2D" mesh = read_mesh(reader, attrs(group)["mesh"]) if haskey(group, "elemorder") diff --git a/src/Spaces/finitedifference.jl b/src/Spaces/finitedifference.jl index 4787c536ed..6d33493cae 100644 --- a/src/Spaces/finitedifference.jl +++ b/src/Spaces/finitedifference.jl @@ -63,10 +63,17 @@ FaceFiniteDifferenceSpace(topology::Topologies.IntervalTopology) = CenterFiniteDifferenceSpace(topology::Topologies.IntervalTopology) = FiniteDifferenceSpace(Grids.FiniteDifferenceGrid(topology), CellCenter()) -FaceFiniteDifferenceSpace(mesh::Meshes.IntervalMesh) = - FiniteDifferenceSpace(Grids.FiniteDifferenceGrid(mesh), CellFace()) -CenterFiniteDifferenceSpace(mesh::Meshes.IntervalMesh) = - FiniteDifferenceSpace(Grids.FiniteDifferenceGrid(mesh), CellCenter()) +FaceFiniteDifferenceSpace( + device::ClimaComms.AbstractDevice, + mesh::Meshes.IntervalMesh, +) = FiniteDifferenceSpace(Grids.FiniteDifferenceGrid(device, mesh), CellFace()) +CenterFiniteDifferenceSpace( + device::ClimaComms.AbstractDevice, + mesh::Meshes.IntervalMesh, +) = FiniteDifferenceSpace( + Grids.FiniteDifferenceGrid(device, mesh), + CellCenter(), +) Adapt.adapt_structure(to, space::FiniteDifferenceSpace) = FiniteDifferenceSpace(Adapt.adapt(to, grid(space)), staggering(space)) diff --git a/src/Topologies/interval.jl b/src/Topologies/interval.jl index 79bfa58f9c..10d1ed5ceb 100644 --- a/src/Topologies/interval.jl +++ b/src/Topologies/interval.jl @@ -55,10 +55,12 @@ function _IntervalTopology( end IntervalTopology(context, mesh, boundaries) end -IntervalTopology(mesh::Meshes.IntervalMesh) = IntervalTopology( - ClimaComms.SingletonCommsContext(ClimaComms.device()), - mesh, -) +# IntervalTopology(mesh::Meshes.IntervalMesh) = IntervalTopology( +# ClimaComms.SingletonCommsContext(ClimaComms.device()), +# mesh, +# ) +IntervalTopology(device::ClimaComms.AbstractDevice, mesh::Meshes.IntervalMesh) = + IntervalTopology(ClimaComms.SingletonCommsContext(device), mesh) isperiodic(topology::AbstractIntervalTopology) = isempty(topology.boundaries) diff --git a/test/Hypsography/2d.jl b/test/Hypsography/2d.jl index c7cd304e47..9d613654ac 100644 --- a/test/Hypsography/2d.jl +++ b/test/Hypsography/2d.jl @@ -19,6 +19,7 @@ using ClimaCore.Geometry FT = Float64 context = ClimaComms.context() +device = ClimaComms.device(context) vertdomain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0), Geometry.ZPoint{FT}(4); @@ -36,7 +37,7 @@ horzdomain = Domains.IntervalDomain( periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = 20) -horztopology = Topologies.IntervalTopology(horzmesh) +horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{4 + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/test/InputOutput/hybrid3dcubedsphere.jl b/test/InputOutput/hybrid3dcubedsphere.jl index a89fa24235..748870b699 100644 --- a/test/InputOutput/hybrid3dcubedsphere.jl +++ b/test/InputOutput/hybrid3dcubedsphere.jl @@ -28,6 +28,7 @@ end z_max = FT(30e3) z_elem = 10 h_elem = 4 + device = ClimaComms.device(comms_ctx) # horizontal space domain = Domains.SphereDomain(R) horizontal_mesh = Meshes.EquiangularCubedSphere(domain, h_elem) @@ -45,7 +46,7 @@ end boundary_names = (:bottom, :top), ) z_mesh = Meshes.IntervalMesh(z_domain, nelems = z_elem) - z_topology = Topologies.IntervalTopology(z_mesh) + z_topology = Topologies.IntervalTopology(device, z_mesh) z_space = Spaces.CenterFiniteDifferenceSpace(z_topology) # Extruded 3D space center_space = Spaces.ExtrudedFiniteDifferenceSpace(h_space, z_space) diff --git a/test/InputOutput/hybrid3dcubedsphere_topography.jl b/test/InputOutput/hybrid3dcubedsphere_topography.jl index e4d205cb92..fece7d85d5 100644 --- a/test/InputOutput/hybrid3dcubedsphere_topography.jl +++ b/test/InputOutput/hybrid3dcubedsphere_topography.jl @@ -28,6 +28,7 @@ end z_max = FT(30e3) z_elem = 10 h_elem = 4 + device = ClimaComms.device(comms_ctx) # horizontal space domain = Domains.SphereDomain(R) horizontal_mesh = Meshes.EquiangularCubedSphere(domain, h_elem) @@ -55,7 +56,7 @@ end ) z_mesh = Meshes.IntervalMesh(z_domain, nelems = z_elem) - z_topology = Topologies.IntervalTopology(z_mesh) + z_topology = Topologies.IntervalTopology(device, z_mesh) z_space = Spaces.CenterFiniteDifferenceSpace(z_topology) # Extruded 3D space center_space = Spaces.ExtrudedFiniteDifferenceSpace( diff --git a/test/Limiters/distributed/dlimiter.jl b/test/Limiters/distributed/dlimiter.jl index 7d753d1574..8eca406f45 100644 --- a/test/Limiters/distributed/dlimiter.jl +++ b/test/Limiters/distributed/dlimiter.jl @@ -47,7 +47,8 @@ zdomain = Domains.IntervalDomain( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(zdomain, nelems = zelems) -vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) +device = ClimaComms.device(context) +vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) quad = Quadratures.GLL{Nij}() horzspace = Spaces.SpectralElementSpace2D(horztopology, quad) diff --git a/test/MatrixFields/field2arrays.jl b/test/MatrixFields/field2arrays.jl index edd4aa00d2..c12de65d12 100644 --- a/test/MatrixFields/field2arrays.jl +++ b/test/MatrixFields/field2arrays.jl @@ -2,6 +2,8 @@ using Test using JET import ClimaCore: Geometry, Domains, Meshes, Spaces, Fields, MatrixFields +import ClimaComms +ClimaComms.@import_required_backends @testset "field2arrays Unit Tests" begin FT = Float64 @@ -11,7 +13,8 @@ import ClimaCore: Geometry, Domains, Meshes, Spaces, Fields, MatrixFields boundary_names = (:bottom, :top), ) mesh = Meshes.IntervalMesh(domain, nelems = 3) - center_space = Spaces.CenterFiniteDifferenceSpace(mesh) + device = ClimaComms.device() + center_space = Spaces.CenterFiniteDifferenceSpace(device, mesh) face_space = Spaces.FaceFiniteDifferenceSpace(center_space) ᶜz = Fields.coordinate_field(center_space).z ᶠz = Fields.coordinate_field(face_space).z diff --git a/test/MatrixFields/matrix_multiplication_recursion.jl b/test/MatrixFields/matrix_multiplication_recursion.jl index 9e9a9917e9..2e504f33d0 100644 --- a/test/MatrixFields/matrix_multiplication_recursion.jl +++ b/test/MatrixFields/matrix_multiplication_recursion.jl @@ -43,7 +43,8 @@ column = ClimaCore.Domains.IntervalDomain( nelements = 150 mesh = ClimaCore.Meshes.IntervalMesh(column; nelems = nelements) -subsurface_space = ClimaCore.Spaces.CenterFiniteDifferenceSpace(mesh) +device = ClimaComms.device() +subsurface_space = ClimaCore.Spaces.CenterFiniteDifferenceSpace(device, mesh) obtain_face_space(cs::ClimaCore.Spaces.CenterFiniteDifferenceSpace) = ClimaCore.Spaces.FaceFiniteDifferenceSpace(cs) function obtain_surface_space(cs::ClimaCore.Spaces.CenterFiniteDifferenceSpace) diff --git a/test/Operators/finitedifference/convergence_advection_diffusion1d.jl b/test/Operators/finitedifference/convergence_advection_diffusion1d.jl index c1a0634fa0..567905a10b 100644 --- a/test/Operators/finitedifference/convergence_advection_diffusion1d.jl +++ b/test/Operators/finitedifference/convergence_advection_diffusion1d.jl @@ -21,6 +21,7 @@ convergence_rate(err, Δh) = FT = Float64 n_elems_seq = 2 .^ (5, 6, 7) err, Δh = zeros(length(n_elems_seq)), zeros(length(n_elems_seq)) + device = ClimaComms.device() for (k, n) in enumerate(n_elems_seq) z₀ = Geometry.ZPoint(FT(0)) @@ -50,7 +51,7 @@ convergence_rate(err, Δh) = mesh = Meshes.IntervalMesh(domain, nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) zc = Fields.coordinate_field(cs) diff --git a/test/Operators/finitedifference/convergence_column.jl b/test/Operators/finitedifference/convergence_column.jl index 2090f2d960..53ed67aee2 100644 --- a/test/Operators/finitedifference/convergence_column.jl +++ b/test/Operators/finitedifference/convergence_column.jl @@ -29,6 +29,7 @@ convergence_rate(err, Δh) = FT = Float64 a, b = FT(0.0), FT(1.0) n_elems_seq = 2 .^ (5, 6, 7, 8) + device = ClimaComms.device() stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(0.5)) for (i, stretch_fn) in enumerate(stretch_fns) err = zeros(FT, length(n_elems_seq)) @@ -43,7 +44,7 @@ convergence_rate(err, Δh) = ) mesh = Meshes.IntervalMesh(domain, stretch_fn, nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) cent_field_exact = zeros(FT, cs) @@ -78,6 +79,7 @@ end FT = Float64 a, b = FT(0.0), FT(1.0) n_elems_seq = 2 .^ (5, 6, 7, 8) + device = ClimaComms.device() stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(0.5)) for (i, stretch_fn) in enumerate(stretch_fns) err, Δh = zeros(FT, length(n_elems_seq)), zeros(FT, length(n_elems_seq)) @@ -90,7 +92,7 @@ end ) mesh = Meshes.IntervalMesh(domain, stretch_fn, nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) face_field_exact = zeros(FT, fs) @@ -130,6 +132,7 @@ end FT = Float64 a, b = FT(0.0), FT(1.0) n_elems_seq = 2 .^ (5, 6, 7, 8) + device = ClimaComms.device() stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(0.5)) for (i, stretch_fn) in enumerate(stretch_fns) err, Δh = zeros(FT, length(n_elems_seq)), zeros(FT, length(n_elems_seq)) @@ -141,7 +144,7 @@ end ) mesh = Meshes.IntervalMesh(domain, stretch_fn, nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) face_field_exact = Geometry.Covariant3Vector.(zeros(FT, fs)) @@ -181,6 +184,7 @@ end FT = Float64 a, b = FT(0.0), FT(1.0) n_elems_seq = 2 .^ (5, 6, 7, 8) + device = ClimaComms.device() stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(0.5)) for (i, stretch_fn) in enumerate(stretch_fns) err, Δh = zeros(FT, length(n_elems_seq)), zeros(FT, length(n_elems_seq)) @@ -192,7 +196,7 @@ end ) mesh = Meshes.IntervalMesh(domain, stretch_fn, nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) cent_field_exact = Geometry.Covariant3Vector.(zeros(FT, cs)) @@ -238,6 +242,7 @@ end err_div_cos_f = zeros(FT, length(n_elems_seq)) err_curl_sin_f = zeros(FT, length(n_elems_seq)) Δh = zeros(FT, length(n_elems_seq)) + device = ClimaComms.device() for (k, n) in enumerate(n_elems_seq) domain = Domains.IntervalDomain( @@ -247,7 +252,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -407,6 +412,7 @@ end err_adv_wc = zeros(FT, length(n_elems_seq)) Δh = zeros(FT, length(n_elems_seq)) + device = ClimaComms.device() for (k, n) in enumerate(n_elems_seq) domain = Domains.IntervalDomain( @@ -416,7 +422,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -457,6 +463,7 @@ end err_adv_wc = zeros(FT, length(n_elems_seq)) Δh = zeros(FT, length(n_elems_seq)) + device = ClimaComms.device() for (k, n) in enumerate(n_elems_seq) domain = Domains.IntervalDomain( @@ -466,7 +473,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -505,6 +512,7 @@ end FT = Float64 n_elems_seq = 2 .^ (4, 6, 8, 10) stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(1.0)) + device = ClimaComms.device() for (i, stretch_fn) in enumerate(stretch_fns) err_adv_wc = zeros(FT, length(n_elems_seq)) @@ -517,7 +525,7 @@ end ) mesh = Meshes.IntervalMesh(domain, stretch_fn; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -566,6 +574,7 @@ end FT = Float64 n_elems_seq = 2 .^ (4, 6, 8, 10) stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(1.0)) + device = ClimaComms.device() for (i, stretch_fn) in enumerate(stretch_fns) err_adv_wc = zeros(FT, length(n_elems_seq)) @@ -578,7 +587,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -625,6 +634,7 @@ end err_adv_wc = zeros(FT, length(n_elems_seq)) Δh = zeros(FT, length(n_elems_seq)) + device = ClimaComms.device() for (k, n) in enumerate(n_elems_seq) domain = Domains.IntervalDomain( @@ -634,7 +644,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -677,6 +687,7 @@ end FT = Float64 n_elems_seq = 2 .^ (4, 6, 8, 10) stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(1.0)) + device = ClimaComms.device() for (i, stretch_fn) in enumerate(stretch_fns) err_adv_wc = zeros(FT, length(n_elems_seq)) @@ -689,7 +700,7 @@ end ) mesh = Meshes.IntervalMesh(domain, stretch_fn; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -747,6 +758,7 @@ end FT = Float64 n_elems_seq = 2 .^ (4, 6, 8, 10) stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(1.0)) + device = ClimaComms.device() for (i, stretch_fn) in enumerate(stretch_fns) err_adv_wc = zeros(FT, length(n_elems_seq)) @@ -759,7 +771,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) centers = getproperty(Fields.coordinate_field(cs), :z) @@ -821,6 +833,7 @@ end n_elems_seq = 2 .^ (5, 6, 7, 8) err = zeros(FT, length(n_elems_seq)) Δh = zeros(FT, length(n_elems_seq)) + device = ClimaComms.device() for (k, n) in enumerate(n_elems_seq) domain = Domains.IntervalDomain( @@ -830,7 +843,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) # advective velocity diff --git a/test/Operators/finitedifference/implicit_stencils_utils.jl b/test/Operators/finitedifference/implicit_stencils_utils.jl index a5bf7dfdd8..1f57339059 100644 --- a/test/Operators/finitedifference/implicit_stencils_utils.jl +++ b/test/Operators/finitedifference/implicit_stencils_utils.jl @@ -62,7 +62,8 @@ function get_space(::Type{FT}) where {FT} hdomain = Domains.SphereDomain(radius) hmesh = Meshes.EquiangularCubedSphere(hdomain, helem) - htopology = Topologies.Topology2D(ClimaComms.SingletonCommsContext(), hmesh) + context = ClimaComms.SingletonCommsContext() + htopology = Topologies.Topology2D(context, hmesh) quad = Quadratures.GLL{npoly + 1}() hspace = Spaces.SpectralElementSpace2D(htopology, quad) @@ -72,8 +73,7 @@ function get_space(::Type{FT}) where {FT} boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = velem) - vtopology = - Topologies.IntervalTopology(ClimaComms.SingletonCommsContext(), vmesh) + vtopology = Topologies.IntervalTopology(context, vmesh) vspace = Spaces.CenterFiniteDifferenceSpace(vtopology) # TODO: Replace this with a space that includes topography. diff --git a/test/Operators/finitedifference/linsolve.jl b/test/Operators/finitedifference/linsolve.jl index 68d2978bbd..2c3f692c09 100644 --- a/test/Operators/finitedifference/linsolve.jl +++ b/test/Operators/finitedifference/linsolve.jl @@ -18,7 +18,9 @@ velem = 4 hdomain = Domains.SphereDomain(radius) hmesh = Meshes.EquiangularCubedSphere(hdomain, helem) -htopology = Topologies.Topology2D(ClimaComms.SingletonCommsContext(), hmesh) +context = ClimaComms.SingletonCommsContext() +device = ClimaComms.device(context) +htopology = Topologies.Topology2D(context, hmesh) quad = Quadratures.GLL{npoly + 1}() hspace = Spaces.SpectralElementSpace2D(htopology, quad) @@ -28,7 +30,7 @@ vdomain = Domains.IntervalDomain( boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = velem) -center_space = Spaces.CenterFiniteDifferenceSpace(vmesh) +center_space = Spaces.CenterFiniteDifferenceSpace(device, vmesh) #= # TODO: Replace this with a space that includes topography. diff --git a/test/Operators/finitedifference/opt.jl b/test/Operators/finitedifference/opt.jl index 61f51a7ddc..33d2cc6f68 100644 --- a/test/Operators/finitedifference/opt.jl +++ b/test/Operators/finitedifference/opt.jl @@ -4,6 +4,8 @@ using JET using IntervalSets import ClimaCore +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: Domains, Meshes, Spaces, Fields, Operators import ClimaCore.Domains: Geometry @@ -206,6 +208,7 @@ end # Test that Julia ia able to optimize Stencil operations v1.7+ @static if @isdefined(var"@test_opt") @testset "Scalar Field FiniteDifferenceSpaces optimizations" begin + device = ClimaComms.device() for FT in (Float64,) domain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0.0), @@ -214,7 +217,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = 16) - center_space = Spaces.CenterFiniteDifferenceSpace(mesh) + center_space = Spaces.CenterFiniteDifferenceSpace(device, mesh) face_space = Spaces.FaceFiniteDifferenceSpace(center_space) faces = getproperty(Fields.coordinate_field(face_space), :z) diff --git a/test/Operators/finitedifference/opt_examples.jl b/test/Operators/finitedifference/opt_examples.jl index 9c6c3de36c..e2cac97405 100644 --- a/test/Operators/finitedifference/opt_examples.jl +++ b/test/Operators/finitedifference/opt_examples.jl @@ -450,7 +450,8 @@ end boundary_names = (:bottom, :top), ) mesh = Meshes.IntervalMesh(domain; nelems = n_elems) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + device = ClimaComms.device() + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) zc = getproperty(Fields.coordinate_field(cs), :z) zf = getproperty(Fields.coordinate_field(fs), :z) diff --git a/test/Operators/finitedifference/opt_implicit_stencils.jl b/test/Operators/finitedifference/opt_implicit_stencils.jl index e2967a19b1..ea2c39bf86 100644 --- a/test/Operators/finitedifference/opt_implicit_stencils.jl +++ b/test/Operators/finitedifference/opt_implicit_stencils.jl @@ -29,7 +29,8 @@ Operators.Operator2Stencil(op::CurriedTwoArgOperator) = hdomain = Domains.SphereDomain(radius) hmesh = Meshes.EquiangularCubedSphere(hdomain, helem) - htopology = Topologies.Topology2D(ClimaComms.SingletonCommsContext(), hmesh) + context = ClimaComms.SingletonCommsContext() + htopology = Topologies.Topology2D(context, hmesh) quad = Quadratures.GLL{npoly + 1}() hspace = Spaces.SpectralElementSpace2D(htopology, quad) @@ -38,8 +39,9 @@ Operators.Operator2Stencil(op::CurriedTwoArgOperator) = Geometry.ZPoint{FT}(zmax); boundary_names = (:bottom, :top), ) + device = ClimaComms.device(context) vmesh = Meshes.IntervalMesh(vdomain, nelems = velem) - vspace = Spaces.CenterFiniteDifferenceSpace(vmesh) + vspace = Spaces.CenterFiniteDifferenceSpace(device, vmesh) # TODO: Replace this with a space that includes topography. center_space = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace) diff --git a/test/Operators/finitedifference/unit_column.jl b/test/Operators/finitedifference/unit_column.jl index 9380ab7533..bb4b13bf4f 100644 --- a/test/Operators/finitedifference/unit_column.jl +++ b/test/Operators/finitedifference/unit_column.jl @@ -125,6 +125,7 @@ end @testset "Test composed stencils" begin are_boundschecks_forced = Base.JLOptions().check_bounds == 1 + device = ClimaComms.device() for FT in (Float32, Float64) domain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0.0), @@ -135,7 +136,7 @@ end mesh = Meshes.IntervalMesh(domain; nelems = 16) - center_space = Spaces.CenterFiniteDifferenceSpace(mesh) + center_space = Spaces.CenterFiniteDifferenceSpace(device, mesh) face_space = Spaces.FaceFiniteDifferenceSpace(center_space) centers = getproperty(Fields.coordinate_field(center_space), :z) @@ -214,6 +215,7 @@ end end @testset "Composite Field FiniteDifferenceSpaces" begin + device = ClimaComms.device() for FT in (Float32, Float64) domain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0.0), @@ -224,7 +226,7 @@ end @test eltype(domain) === Geometry.ZPoint{FT} mesh = Meshes.IntervalMesh(domain; nelems = 16) - center_space = Spaces.CenterFiniteDifferenceSpace(mesh) + center_space = Spaces.CenterFiniteDifferenceSpace(device, mesh) face_space = Spaces.FaceFiniteDifferenceSpace(center_space) FieldType = NamedTuple{(:a, :b), Tuple{FT, FT}} @@ -244,6 +246,7 @@ end @testset "Biased interpolation" begin FT = Float64 n_elems = 10 + device = ClimaComms.device() domain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0.0), @@ -252,7 +255,7 @@ end ) mesh = Meshes.IntervalMesh(domain; nelems = n_elems) - cs = Spaces.CenterFiniteDifferenceSpace(mesh) + cs = Spaces.CenterFiniteDifferenceSpace(device, mesh) fs = Spaces.FaceFiniteDifferenceSpace(cs) zc = getproperty(Fields.coordinate_field(cs), :z) diff --git a/test/Operators/finitedifference/wfact.jl b/test/Operators/finitedifference/wfact.jl index 4c53e26b32..7bc4ac29c4 100644 --- a/test/Operators/finitedifference/wfact.jl +++ b/test/Operators/finitedifference/wfact.jl @@ -28,7 +28,8 @@ velem = 4 hdomain = Domains.SphereDomain(radius) hmesh = Meshes.EquiangularCubedSphere(hdomain, helem) -htopology = Topologies.Topology2D(ClimaComms.SingletonCommsContext(), hmesh) +context = ClimaComms.SingletonCommsContext() +htopology = Topologies.Topology2D(context, hmesh) quad = Quadratures.GLL{npoly + 1}() hspace = Spaces.SpectralElementSpace2D(htopology, quad) @@ -38,7 +39,8 @@ vdomain = Domains.IntervalDomain( boundary_names = (:bottom, :top), ) vmesh = Meshes.IntervalMesh(vdomain, nelems = velem) -center_space = Spaces.CenterFiniteDifferenceSpace(vmesh) +device = ClimaComms.device(context) +center_space = Spaces.CenterFiniteDifferenceSpace(device, vmesh) #= # TODO: Replace this with a space that includes topography. diff --git a/test/Operators/hybrid/dss_opt.jl b/test/Operators/hybrid/dss_opt.jl index abe53d21e5..1de4cdd354 100644 --- a/test/Operators/hybrid/dss_opt.jl +++ b/test/Operators/hybrid/dss_opt.jl @@ -27,14 +27,13 @@ vertdomain = Domains.IntervalDomain( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = 10) -vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) +device = ClimaComms.CPUSingleThreaded() +vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.SphereDomain(30.0) horzmesh = Meshes.EquiangularCubedSphere(horzdomain, 4) -horztopology = Topologies.Topology2D( - ClimaComms.SingletonCommsContext(ClimaComms.CPUSingleThreaded()), - horzmesh, -) +horztopology = + Topologies.Topology2D(ClimaComms.SingletonCommsContext(device), horzmesh) quad = Quadratures.GLL{5}() horzspace = Spaces.SpectralElementSpace2D(horztopology, quad) diff --git a/test/Operators/hybrid/opt.jl b/test/Operators/hybrid/opt.jl index 52c054dbd5..574558cde8 100644 --- a/test/Operators/hybrid/opt.jl +++ b/test/Operators/hybrid/opt.jl @@ -211,7 +211,8 @@ function hspace1d(FT) periodic = true, ) hmesh = Meshes.IntervalMesh(hdomain, nelems = 3) - htopology = Topologies.IntervalTopology(hmesh) + device = ClimaComms.device() + htopology = Topologies.IntervalTopology(device, hmesh) Nq = 3 quad = Quadratures.GLL{Nq}() return Spaces.SpectralElementSpace1D(htopology, quad) @@ -237,6 +238,7 @@ end @static if @isdefined(var"@test_opt") @testset "Scalar Field ExtrudedFiniteDifferenceSpace" begin + device = ClimaComms.device() for FT in (Float64,), hspace in (hspace1d(FT), hspace2d(FT)) vdomain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0.0), @@ -244,7 +246,7 @@ end boundary_names = (:left, :right), ) vmesh = Meshes.IntervalMesh(vdomain; nelems = 16) - cvspace = Spaces.CenterFiniteDifferenceSpace(vmesh) + cvspace = Spaces.CenterFiniteDifferenceSpace(device, vmesh) center_space = Spaces.ExtrudedFiniteDifferenceSpace(hspace, cvspace) face_space = Spaces.FaceExtrudedFiniteDifferenceSpace(center_space) diff --git a/test/Operators/hybrid/utils_2d.jl b/test/Operators/hybrid/utils_2d.jl index bb2f5bd9f2..8ff92e7cdc 100644 --- a/test/Operators/hybrid/utils_2d.jl +++ b/test/Operators/hybrid/utils_2d.jl @@ -15,6 +15,8 @@ using Test using StaticArrays, IntervalSets, LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends import ClimaCore: ClimaCore, @@ -50,14 +52,15 @@ function hvspace_2D(; boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, stretch, nelems = velem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + device = ClimaComms.device() + vert_center_space = Spaces.CenterFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]) .. Geometry.XPoint{FT}(xlim[2]), periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = helem) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{npoly + 1}() horzspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/test/Operators/remapping.jl b/test/Operators/remapping.jl index 280d460e8a..0301c377fa 100644 --- a/test/Operators/remapping.jl +++ b/test/Operators/remapping.jl @@ -18,9 +18,10 @@ using IntervalSets, LinearAlgebra, SparseArrays FT = Float64 function make_space(domain::Domains.IntervalDomain, nq, nelems = 1) + device = ClimaComms.CPUSingleThreaded() nq == 1 ? (quad = Quadratures.GL{1}()) : (quad = Quadratures.GLL{nq}()) mesh = Meshes.IntervalMesh(domain; nelems = nelems) - topo = Topologies.IntervalTopology(mesh) + topo = Topologies.IntervalTopology(device, mesh) space = Spaces.SpectralElementSpace1D(topo, quad) return space end @@ -33,10 +34,9 @@ function make_space( ) nq == 1 ? (quad = Quadratures.GL{1}()) : (quad = Quadratures.GLL{nq}()) mesh = Meshes.RectilinearMesh(domain, nxelems, nyelems) - topology = Topologies.Topology2D( - ClimaComms.SingletonCommsContext(ClimaComms.CPUSingleThreaded()), - mesh, - ) + device = ClimaComms.CPUSingleThreaded() + topology = + Topologies.Topology2D(ClimaComms.SingletonCommsContext(device), mesh) space = Spaces.SpectralElementSpace2D(topology, quad) return space end diff --git a/test/Operators/spectralelement/opt.jl b/test/Operators/spectralelement/opt.jl index 697d2054c6..c2b93468f5 100644 --- a/test/Operators/spectralelement/opt.jl +++ b/test/Operators/spectralelement/opt.jl @@ -179,6 +179,7 @@ end end @testset "Spectral Element 3D Hybrid Field optimizations" begin + device = ClimaComms.CPUSingleThreaded() for FT in (Float64, Float32) xelem = 3 yelem = 3 @@ -191,7 +192,8 @@ end boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = velem) - vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh) + vert_center_space = + Spaces.CenterFiniteDifferenceSpace(device, vertmesh) xdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(-500) .. Geometry.XPoint{FT}(500), @@ -205,9 +207,7 @@ end horzdomain = Domains.RectangleDomain(xdomain, ydomain) horzmesh = Meshes.RectilinearMesh(horzdomain, xelem, yelem) horztopology = Topologies.Topology2D( - ClimaComms.SingletonCommsContext( - ClimaComms.CPUSingleThreaded(), - ), + ClimaComms.SingletonCommsContext(device), horzmesh, ) diff --git a/test/Spaces/opt_spaces.jl b/test/Spaces/opt_spaces.jl index 2315f6b6f5..f8ea1cd2c2 100644 --- a/test/Spaces/opt_spaces.jl +++ b/test/Spaces/opt_spaces.jl @@ -36,20 +36,20 @@ end test_n_failures(86, TU.PointSpace, context) test_n_failures(144, TU.SpectralElementSpace1D, context) test_n_failures(1106, TU.SpectralElementSpace2D, context) - test_n_failures(4, TU.ColumnCenterFiniteDifferenceSpace, context) - test_n_failures(5, TU.ColumnFaceFiniteDifferenceSpace, context) + test_n_failures(123, TU.ColumnCenterFiniteDifferenceSpace, context) + test_n_failures(123, TU.ColumnFaceFiniteDifferenceSpace, context) test_n_failures(1112, TU.SphereSpectralElementSpace, context) - test_n_failures(1114, TU.CenterExtrudedFiniteDifferenceSpace, context) - test_n_failures(1114, TU.FaceExtrudedFiniteDifferenceSpace, context) + test_n_failures(1139, TU.CenterExtrudedFiniteDifferenceSpace, context) + test_n_failures(1139, TU.FaceExtrudedFiniteDifferenceSpace, context) else test_n_failures(0, TU.PointSpace, context) test_n_failures(137, TU.SpectralElementSpace1D, context) test_n_failures(272, TU.SpectralElementSpace2D, context) - test_n_failures(4, TU.ColumnCenterFiniteDifferenceSpace, context) - test_n_failures(5, TU.ColumnFaceFiniteDifferenceSpace, context) + test_n_failures(118, TU.ColumnCenterFiniteDifferenceSpace, context) + test_n_failures(118, TU.ColumnFaceFiniteDifferenceSpace, context) test_n_failures(278, TU.SphereSpectralElementSpace, context) - test_n_failures(283, TU.CenterExtrudedFiniteDifferenceSpace, context) - test_n_failures(283, TU.FaceExtrudedFiniteDifferenceSpace, context) + test_n_failures(321, TU.CenterExtrudedFiniteDifferenceSpace, context) + test_n_failures(321, TU.FaceExtrudedFiniteDifferenceSpace, context) # The OBJECT_CACHE causes inference failures that inhibit understanding # inference failures in _SpectralElementGrid2D, so let's `@test_opt` those diff --git a/test/Spaces/sphere.jl b/test/Spaces/sphere.jl index 17305d9536..4db6ad1760 100644 --- a/test/Spaces/sphere.jl +++ b/test/Spaces/sphere.jl @@ -98,8 +98,9 @@ end Geometry.ZPoint{FT}(zlim[2]); boundary_names = (:bottom, :top), ) + device = ClimaComms.device(context) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem) - vertgrid = Grids.FiniteDifferenceGrid(vertmesh) + vertgrid = Grids.FiniteDifferenceGrid(device, vertmesh) horzdomain = Domains.SphereDomain(radius) horzmesh = Meshes.EquiangularCubedSphere(horzdomain, helem) diff --git a/test/Spaces/terrain_warp.jl b/test/Spaces/terrain_warp.jl index 6919487091..f1ac451fe2 100644 --- a/test/Spaces/terrain_warp.jl +++ b/test/Spaces/terrain_warp.jl @@ -60,7 +60,7 @@ function generate_base_spaces_2d( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, stretch, nelems = velem) - vert_face_space = Spaces.FaceFiniteDifferenceSpace(vertmesh) + vert_face_space = Spaces.FaceFiniteDifferenceSpace(device, vertmesh) # Generate Horizontal Space quad = Quadratures.GLL{npoly + 1}() @@ -91,7 +91,7 @@ function generate_base_spaces_3d( boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, stretch, nelems = velem) - vert_face_space = Spaces.FaceFiniteDifferenceSpace(vertmesh) + vert_face_space = Spaces.FaceFiniteDifferenceSpace(device, vertmesh) # Generate Horizontal Space quad = Quadratures.GLL{npoly + 1}() @@ -480,7 +480,7 @@ end boundary_names = (:bottom, :top), ) vertmesh = Meshes.IntervalMesh(vertdomain, nelems = nl) - vert_face_space = Spaces.FaceFiniteDifferenceSpace(vertmesh) + vert_face_space = Spaces.FaceFiniteDifferenceSpace(device, vertmesh) horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(xlim[1]), @@ -488,7 +488,7 @@ end periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain, nelems = nh) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{np + 1}() hspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/test/Spaces/unit_spaces.jl b/test/Spaces/unit_spaces.jl index 6c47a2eb80..dee58809ee 100644 --- a/test/Spaces/unit_spaces.jl +++ b/test/Spaces/unit_spaces.jl @@ -1,3 +1,7 @@ +#= +julia --project +using Revise; include(joinpath("test", "Spaces", "unit_spaces.jl")) +=# using Test using ClimaComms using StaticArrays, IntervalSets, LinearAlgebra @@ -27,15 +31,15 @@ on_gpu = ClimaComms.device() isa ClimaComms.CUDADevice Geometry.XPoint{FT}(-3) .. Geometry.XPoint{FT}(5), periodic = true, ) + device = ClimaComms.device() mesh = Meshes.IntervalMesh(domain; nelems = 1) - topology = Topologies.IntervalTopology(mesh) + topology = Topologies.IntervalTopology(device, mesh) quad = Quadratures.GLL{4}() points, weights = Quadratures.quadrature_points(FT, quad) space = Spaces.SpectralElementSpace1D(topology, quad) - device = ClimaComms.device() expected_repr = """ SpectralElementSpace1D: @@ -85,6 +89,7 @@ on_gpu || @testset "extruded (2d 1×3) finite difference space" begin FT = Float32 + device = ClimaComms.device() vertdomain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0), Geometry.ZPoint{FT}(10); @@ -92,7 +97,7 @@ on_gpu || @testset "extruded (2d 1×3) finite difference space" begin ) vertmesh = Meshes.IntervalMesh(vertdomain, Meshes.Uniform(), nelems = 10) - vert_face_space = Spaces.FaceFiniteDifferenceSpace(vertmesh) + vert_face_space = Spaces.FaceFiniteDifferenceSpace(device, vertmesh) # Generate Horizontal Space horzdomain = Domains.IntervalDomain( Geometry.XPoint{FT}(0), @@ -100,7 +105,7 @@ on_gpu || @testset "extruded (2d 1×3) finite difference space" begin periodic = true, ) horzmesh = Meshes.IntervalMesh(horzdomain; nelems = 5) - horztopology = Topologies.IntervalTopology(horzmesh) + horztopology = Topologies.IntervalTopology(device, horzmesh) quad = Quadratures.GLL{4}() hspace = Spaces.SpectralElementSpace1D(horztopology, quad) diff --git a/test/Topologies/interval.jl b/test/Topologies/interval.jl index c9643e87c7..0578845dec 100644 --- a/test/Topologies/interval.jl +++ b/test/Topologies/interval.jl @@ -1,5 +1,6 @@ using Test using ClimaCore: Geometry, Domains, Meshes, Topologies +using ClimaComms # need to make sure mesh objects with different arrays but same contents give identical topologies # https://github.com/CliMA/ClimaCore.jl/issues/1592 @@ -16,8 +17,8 @@ mesh2 = Meshes.IntervalMesh(domain, [Geometry.ZPoint(Float64(i)) for i in 0:10]) @test mesh1 == mesh2 @test isequal(mesh1, mesh2) @test hash(mesh1) == hash(mesh2) - -topology1 = Topologies.IntervalTopology(mesh1) -topology2 = Topologies.IntervalTopology(mesh2) +device = ClimaComms.device() +topology1 = Topologies.IntervalTopology(device, mesh1) +topology2 = Topologies.IntervalTopology(device, mesh2) @test topology1 === topology2