From b3ba6cdc93a7e66b3e3c7e5c64137e60dcc6dd00 Mon Sep 17 00:00:00 2001
From: Charles Kawczynski <kawczynski.charles@gmail.com>
Date: Fri, 1 Dec 2023 09:17:30 -0500
Subject: [PATCH] Add gpu box limiters job

---
 .buildkite/pipeline.yml                   |  9 +++++++++
 examples/hybrid/box/limiters_advection.jl | 12 +++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 6819acf276..1c3a571caa 100755
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -987,6 +987,15 @@ steps:
         artifact_paths:
           - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*"
 
+      - label: ":computer: 3D Box limiters advection Gaussian bells GPU"
+        key: "gpu_box_advection_limiter_gaussian_bells"
+        command:
+          - "julia --color=yes --project=examples examples/hybrid/box/limiters_advection.jl gaussian_bells"
+        artifact_paths:
+          - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*"
+        agents:
+          slurm_gpus: 1
+
       - label: ":computer: 3D Box limiters advection slotted spheres"
         key: "cpu_box_advection_limiter_slotted_spheres"
         command:
diff --git a/examples/hybrid/box/limiters_advection.jl b/examples/hybrid/box/limiters_advection.jl
index 486e8fff99..16c71e2778 100644
--- a/examples/hybrid/box/limiters_advection.jl
+++ b/examples/hybrid/box/limiters_advection.jl
@@ -45,7 +45,7 @@ convergence_rate(err, Δh) =
 # Function space setup
 function hvspace_3D(
     ::Type{FT};
-    device = ClimaComms.device(),
+    device,
     context = ClimaComms.SingletonCommsContext(device),
     xlim = (-2π, 2π),
     ylim = (-2π, 2π),
@@ -149,8 +149,11 @@ function init_state(cspace, fspace, test_case, params)
     # Initialize state
     ρ_init = ρ₀ .* ones(cspace)
     q_init = map(Fields.coordinate_field(cspace)) do coord
+        rd = (
+            Geometry.euclidean_distance(coord, bell_centers[1]),
+            Geometry.euclidean_distance(coord, bell_centers[2]),
+        )
         (; x, y, z) = coord
-        rd = Geometry.euclidean_distance.((coord,), bell_centers)
         if test_case isa SlottedSpheres
             if rd[1] <= r0 && abs(x - bell_centers[1].x) >= r0 / 6
                 return 1.0
@@ -187,7 +190,9 @@ end
 ENV["GKSwstype"] = "nul"
 using ClimaCorePlots, Plots
 Plots.GRBackend()
-dirname = "box_advection_limiter_$(name(test_case))"
+device = ClimaComms.device()
+dev_suffix = device isa ClimaComms.CUDADevice ? "_gpu" : ""
+dirname = "box_advection_limiter_$(name(test_case))$dev_suffix"
 
 FT = Float64;
 params = LimAdvectionParams{FT}();
@@ -282,6 +287,7 @@ for (k, horz_ne) in enumerate(horz_ne_seq)
     # Set up 3D spatial domain - doubly periodic box
     cspace, fspace = hvspace_3D(
         FT;
+        device,
         Nij = Nij,
         xelems = horz_ne,
         yelems = horz_ne,