From 25e8f2139be1f41fc20a857995c581508cd1a112 Mon Sep 17 00:00:00 2001
From: Simon Byrne <simonbyrne@gmail.com>
Date: Sun, 17 Sep 2023 05:44:28 -0700
Subject: [PATCH] inline all FD operators and axisvector conversions

---
 src/Geometry/axistensors.jl       | 11 +++++----
 src/Geometry/conversions.jl       | 40 ++++++++++++++++++-------------
 src/Operators/finitedifference.jl |  2 +-
 3 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/src/Geometry/axistensors.jl b/src/Geometry/axistensors.jl
index ecc33598ba..f342ec89a7 100644
--- a/src/Geometry/axistensors.jl
+++ b/src/Geometry/axistensors.jl
@@ -609,10 +609,13 @@ end
             push!(vals, val)
         end
     end
-    return :(@inbounds Axis2Tensor(
-        (ato, axes(x, 2)),
-        SMatrix{$(length(Ito)), $M}($(vals...)),
-    ))
+    quote
+        Base.@_propagate_inbounds_meta
+        @inbounds Axis2Tensor(
+            (ato, axes(x, 2)),
+            SMatrix{$(length(Ito)), $M}($(vals...)),
+        )
+    end
 end
 
 @inline transform(ato::CovariantAxis, v::CovariantTensor) = _project(ato, v)
diff --git a/src/Geometry/conversions.jl b/src/Geometry/conversions.jl
index 5330808a4a..af9ff39add 100644
--- a/src/Geometry/conversions.jl
+++ b/src/Geometry/conversions.jl
@@ -49,65 +49,73 @@ CovariantVector(
 ) where {T, I} = local_geometry.gᵢⱼ * u
 
 # Converting to specific dimension types
-(::Type{<:ContravariantVector{<:Any, I}})(
+@inline (::Type{<:ContravariantVector{<:Any, I}})(
     u::ContravariantVector{<:Any, I},
     ::LocalGeometry{I},
 ) where {I} = u
 
-(::Type{<:ContravariantVector{<:Any, I}})(
+@inline (::Type{<:ContravariantVector{<:Any, I}})(
     u::ContravariantVector,
     ::LocalGeometry,
 ) where {I} = project(ContravariantAxis{I}(), u)
 
-(::Type{<:ContravariantVector{<:Any, I}})(
+@inline (::Type{<:ContravariantVector{<:Any, I}})(
     u::AxisVector,
     local_geometry::LocalGeometry,
 ) where {I} =
     project(ContravariantAxis{I}(), ContravariantVector(u, local_geometry))
 
-(::Type{<:CovariantVector{<:Any, I}})(
+@inline (::Type{<:CovariantVector{<:Any, I}})(
     u::CovariantVector{<:Any, I},
     ::LocalGeometry{I},
 ) where {I} = u
 
-(::Type{<:CovariantVector{<:Any, I}})(
+@inline (::Type{<:CovariantVector{<:Any, I}})(
     u::CovariantVector,
     ::LocalGeometry,
 ) where {I} = project(CovariantAxis{I}(), u)
 
-(::Type{<:CovariantVector{<:Any, I}})(
+@inline (::Type{<:CovariantVector{<:Any, I}})(
     u::AxisVector,
     local_geometry::LocalGeometry,
 ) where {I} = project(CovariantAxis{I}(), CovariantVector(u, local_geometry))
 
-(::Type{<:LocalVector{<:Any, I}})(
+@inline (::Type{<:LocalVector{<:Any, I}})(
     u::LocalVector{<:Any, I},
     ::LocalGeometry{I},
 ) where {I} = u
 
-(::Type{<:LocalVector{<:Any, I}})(u::LocalVector, ::LocalGeometry) where {I} =
-    project(LocalAxis{I}(), u)
+@inline (::Type{<:LocalVector{<:Any, I}})(
+    u::LocalVector,
+    ::LocalGeometry,
+) where {I} = project(LocalAxis{I}(), u)
 
-(::Type{<:LocalVector{<:Any, I}})(
+@inline (::Type{<:LocalVector{<:Any, I}})(
     u::AxisVector,
     local_geometry::LocalGeometry,
 ) where {I} = project(LocalAxis{I}(), LocalVector(u, local_geometry))
 
 # Generic N-axis conversion functions,
 # Convert to specific local geometry dimension then convert vector type
-LocalVector(u::CovariantVector, local_geometry::LocalGeometry{I}) where {I} =
+@inline LocalVector(
+    u::CovariantVector,
+    local_geometry::LocalGeometry{I},
+) where {I} =
     project(LocalAxis{I}(), project(CovariantAxis{I}(), u), local_geometry)
 
-LocalVector(
+@inline LocalVector(
     u::ContravariantVector,
     local_geometry::LocalGeometry{I},
 ) where {I} =
     project(LocalAxis{I}(), project(ContravariantAxis{I}(), u), local_geometry)
 
-CovariantVector(u::LocalVector, local_geometry::LocalGeometry{I}) where {I} =
+@inline CovariantVector(
+    u::LocalVector,
+    local_geometry::LocalGeometry{I},
+) where {I} =
     project(CovariantAxis{I}(), project(LocalAxis{I}(), u), local_geometry)
 
-CovariantVector(
+@inline CovariantVector(
     u::ContravariantVector,
     local_geometry::LocalGeometry{I},
 ) where {I} = project(
@@ -116,13 +124,13 @@ CovariantVector(
     local_geometry,
 )
 
-ContravariantVector(
+@inline ContravariantVector(
     u::LocalVector,
     local_geometry::LocalGeometry{I},
 ) where {I} =
     project(ContravariantAxis{I}(), project(LocalAxis{I}(), u), local_geometry)
 
-ContravariantVector(
+@inline ContravariantVector(
     u::CovariantVector,
     local_geometry::LocalGeometry{I},
 ) where {I} = project(
diff --git a/src/Operators/finitedifference.jl b/src/Operators/finitedifference.jl
index e137846a12..31d3d6edc1 100644
--- a/src/Operators/finitedifference.jl
+++ b/src/Operators/finitedifference.jl
@@ -3446,7 +3446,7 @@ function Base.copyto!(
     max_threads = 256
     nitems = Nv * Nq * Nq * Nh # # of independent items
     (nthreads, nblocks) = Spaces._configure_threadblock(max_threads, nitems)
-    @cuda threads = (nthreads,) blocks = (nblocks,) copyto_stencil_kernel!(
+    @cuda always_inline = true threads = (nthreads,) blocks = (nblocks,) copyto_stencil_kernel!(
         strip_space(out, space),
         strip_space(bc, space),
         axes(out),