From 5110a63b806b8eaa4e7930f8b3c40056f4b25d0e Mon Sep 17 00:00:00 2001
From: chapman39 <chapman39@llnl.gov>
Date: Wed, 9 Oct 2024 11:54:55 -0700
Subject: [PATCH 01/19] Create run benchmarks script and setup in CI

---
 .gitlab-ci.yml                 |  24 +++++++-
 .gitlab/build_blueos.yml       |  12 ++++
 .gitlab/build_toss4.yml        |  22 ++++++-
 .gitlab/build_toss4_cray.yml   |  15 ++++-
 scripts/llnl/run_benchmarks.py | 109 +++++++++++++++++++++++++++++++++
 5 files changed, 175 insertions(+), 7 deletions(-)
 create mode 100755 scripts/llnl/run_benchmarks.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 70a23d529..5a8463f92 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,11 +19,15 @@ variables:
 
 .src_workflow:
   rules:
-    - if: '$FULL_BUILD != "ON"'
+    - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD != "ON"'
 
 .full_workflow:
   rules:
-    - if: '$FULL_BUILD == "ON"'
+    - if: '$FULL_BUILD == "ON" && $BENCHMARKS_BUILD != "ON"'
+
+.benchmarks_workflow:
+  rules:
+    - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD == "ON"'
 
 ####
 # Templates
@@ -66,6 +70,22 @@ variables:
     reports:
       junit: ${FULL_BUILD_ROOT}/${SYS_TYPE}/*/_serac_build_and_test_*/build-*/junit.xml
 
+
+.benchmarks_build_script:
+  script:
+    # Builds src, runs benchmarks, and stores Caliper files in shared location
+    - echo -e "section_start:$(date +%s):benchmarks_build\r\e[0K
+      Benchmarks Build ${CI_PROJECT_NAME}"
+    - ${ALLOC_COMMAND} python3 scripts/llnl/run_benchmarks.py --host-config ${HOST_CONFIG} --extra-cmake-options ${EXTRA_CMAKE_OPTIONS}
+    - echo -e "section_end:$(date +%s):benchmarks_build\r\e[0K"
+  artifacts:
+    expire_in: 2 weeks
+    when: always
+    paths:
+      - _serac_build_and_test_*/output.log*.txt
+      - _serac_build_and_test_*/build-*/output.log*.txt
+      - _serac_build_and_test_*/build-*/*.cali
+
 # This is where jobs are included for each system
 include:
   - local: .gitlab/build_blueos.yml
diff --git a/.gitlab/build_blueos.yml b/.gitlab/build_blueos.yml
index eff30dd57..2cb47e3ba 100644
--- a/.gitlab/build_blueos.yml
+++ b/.gitlab/build_blueos.yml
@@ -37,6 +37,10 @@
   extends: [.full_build_script, .on_blueos, .full_workflow]
   needs: []
 
+.benchmarks_build_on_blueos:
+  extends: [.benchmarks_build_script, .on_blueos, .benchmarks_workflow]
+  needs: []
+
 ####
 # Build jobs
 blueos-clang_10_0_1-src:
@@ -58,3 +62,11 @@ blueos-clang_10_0_1-full:
     ALLOC_NODES: "1"
     ALLOC_TIME: "55"
   extends: [.full_build_on_blueos, .with_cuda]
+
+blueos-clang_10_0_1-benchmarks:
+  variables:
+    COMPILER: "clang@10.0.1"
+    HOST_CONFIG: "lassen-blueos_3_ppc64le_ib_p9-${COMPILER}_cuda.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: [.benchmarks_build_on_blueos, .with_cuda]
diff --git a/.gitlab/build_toss4.yml b/.gitlab/build_toss4.yml
index c4f83eabc..58d79ed54 100644
--- a/.gitlab/build_toss4.yml
+++ b/.gitlab/build_toss4.yml
@@ -28,6 +28,10 @@
     # LC version of pip is ancient
     - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
 
+.benchmarks_build_on_toss4:
+  extends: [.benchmarks_build_script, .on_toss4, .benchmarks_workflow]
+  needs: []
+
 
 ####
 # Build jobs
@@ -76,7 +80,6 @@ toss4-clang_14_0_6-full:
     SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
   extends: .full_build_on_toss4
 
 toss4-gcc_10_3_1-full:
@@ -85,5 +88,20 @@ toss4-gcc_10_3_1-full:
     SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
   extends: .full_build_on_toss4
+
+toss4-clang_14_0_6-benchmarks:
+  variables:
+    COMPILER: "clang@14.0.6"
+    HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: .benchmarks_build_on_toss4
+
+toss4-gcc_10_3_1-benchmarks:
+  variables:
+    COMPILER: "gcc@10.3.1"
+    HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: .benchmarks_build_on_toss4
diff --git a/.gitlab/build_toss4_cray.yml b/.gitlab/build_toss4_cray.yml
index 14d77d083..17c0bf6c9 100644
--- a/.gitlab/build_toss4_cray.yml
+++ b/.gitlab/build_toss4_cray.yml
@@ -25,6 +25,10 @@
     # LC version of pip is ancient
     - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
 
+.benchmarks_build_on_toss4_cray:
+  extends: [.benchmarks_build_script, .on_toss4_cray, .benchmarks_workflow]
+  needs: []
+
 
 ####
 # Build jobs
@@ -41,9 +45,14 @@ toss4_cray-clang_17_0_0-src:
 
 toss4_cray-clang_17_0_0-full:
   variables:
-    COMPILER: "clang@17.0.0"
-    SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
   extends: .full_build_on_toss4_cray
+
+toss4_cray-clang_17_0_0-benchmarks:
+  variables:
+    COMPILER: "clang@17.0.0"
+    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: .benchmarks_build_on_toss4_cray
diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py
new file mode 100755
index 000000000..178b69ffb
--- /dev/null
+++ b/scripts/llnl/run_benchmarks.py
@@ -0,0 +1,109 @@
+#!/bin/sh
+"exec" "python3" "-u" "-B" "$0" "$@"
+
+# Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and
+# other Serac Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+
+"""
+ file: run_benchmarks.py
+
+ description: 
+  Run benchmarks and update shared (or any desired) location with new Caliper files
+
+"""
+
+from common_build_functions import *
+
+from argparse import ArgumentParser
+
+import os
+
+
+def parse_args():
+    "Parses args from command line"
+    parser = ArgumentParser()
+    parser.add_argument("-e", "--extra-cmake-options",
+                      dest="extra_cmake_options",
+                      default="",
+                      help="Extra cmake options to add to the cmake configure line. Note '-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF' is always appended.")
+    parser.add_argument("-hc", "--host-config",
+                    dest="host_config",
+                    default=os.environ.get("HOST_CONFIG", None),
+                    help="Specific host-config file to build (defaults to HOST_CONFIG environment variable)")
+    parser.add_argument("-sd", "--spot-directory",
+                      dest="spot_dir",
+                      default=get_shared_spot_dir(),
+                      help="Where to put all resulting caliper files to use for SPOT analysis (defaults to a shared location)")
+    parser.add_argument("-t", "--timestamp",
+                    dest="timestamp",
+                    default=get_timestamp(),
+                    help="Set timestamp manually for debugging")
+
+    # Parse args
+    args, extra_args = parser.parse_known_args()
+    args = vars(args)
+
+    # Verify args
+    if args["host_config"] is None:
+        print("[ERROR: Both host_config argument and HOST_CONFIG environment variable unset!]")
+        sys.exit(1)
+
+    if not os.path.exists(args["host_config"]):
+        print("[ERROR: Host config path does not exist: %s]" % args["host_config"])
+        sys.exit(1)
+
+    return args
+
+
+def main():
+    # Args
+    args = parse_args()
+    cmake_options = args["extra_cmake_options"] + " -DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF"
+    host_config = args["host_config"]
+    spot_dir = args["spot_dir"]
+    timestamp = args["timestamp"]
+
+    # Vars
+    repo_dir = get_repo_dir()
+    test_root = get_build_and_test_root(repo_dir, timestamp)
+    host_config_root = get_host_config_root(host_config)
+    benchmarks_output_file = os.path.join(test_root, "output.log.%s.benchmarks.txt" % host_config_root)
+
+    # Build Serac
+    os.chdir(repo_dir)
+    os.makedirs(test_root, exist_ok=True)
+    build_and_test_host_config(test_root=test_root, host_config=host_config,
+                               report_to_stdout=True, extra_cmake_options=cmake_options,
+                               skip_install=True, skip_tests=True)
+
+    # Go to build location
+    build_dir=""
+    dirs = glob.glob(pjoin(test_root, "*"))
+    for dir in dirs:
+        if os.path.exists(dir) and "build-" in dir:
+            build_dir=dir
+    os.chdir(build_dir)
+
+    # Run benchmarks
+    shell_exec("make run_benchmarks", echo=True, print_output=True, output_file=benchmarks_output_file)
+
+    # Move resulting .cali files to specified directory
+    os.makedirs(spot_dir, exist_ok=True)
+    cali_files = glob.glob(pjoin(build_dir, "*.cali"))
+    for cali_file in cali_files:
+        if os.path.exists(cali_file):
+            shutil.copy2(cali_file, spot_dir)
+
+    # Print SPOT url
+    if on_rz():
+        print("[View SPOT directory here: https://rzlc.llnl.gov/spot2/?sf={0}]".format(spot_dir))
+    else:
+        print("[View SPOT directory here: https://lc.llnl.gov/spot2/?sf={0}]".format(spot_dir))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

From f1e69161b9b1601a69ad86288fcc861b8fcaf559 Mon Sep 17 00:00:00 2001
From: chapman39 <chapman39@llnl.gov>
Date: Wed, 9 Oct 2024 12:54:34 -0700
Subject: [PATCH 02/19] Use environment variables for host configs and cmake
 args

---
 .gitlab-ci.yml                 | 2 +-
 scripts/llnl/run_benchmarks.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5a8463f92..2ea9506aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -76,7 +76,7 @@ variables:
     # Builds src, runs benchmarks, and stores Caliper files in shared location
     - echo -e "section_start:$(date +%s):benchmarks_build\r\e[0K
       Benchmarks Build ${CI_PROJECT_NAME}"
-    - ${ALLOC_COMMAND} python3 scripts/llnl/run_benchmarks.py --host-config ${HOST_CONFIG} --extra-cmake-options ${EXTRA_CMAKE_OPTIONS}
+    - ${ALLOC_COMMAND} python3 scripts/llnl/run_benchmarks.py
     - echo -e "section_end:$(date +%s):benchmarks_build\r\e[0K"
   artifacts:
     expire_in: 2 weeks
diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py
index 178b69ffb..aef34bf62 100755
--- a/scripts/llnl/run_benchmarks.py
+++ b/scripts/llnl/run_benchmarks.py
@@ -26,7 +26,7 @@ def parse_args():
     parser = ArgumentParser()
     parser.add_argument("-e", "--extra-cmake-options",
                       dest="extra_cmake_options",
-                      default="",
+                      default=os.environ.get("EXTRA_CMAKE_OPTIONS", ""),
                       help="Extra cmake options to add to the cmake configure line. Note '-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF' is always appended.")
     parser.add_argument("-hc", "--host-config",
                     dest="host_config",

From a6702b1c2a0ef460ddf2ae60fc407703811df6cb Mon Sep 17 00:00:00 2001
From: chapman39 <chapman39@llnl.gov>
Date: Wed, 9 Oct 2024 13:17:14 -0700
Subject: [PATCH 03/19] find host config path

---
 scripts/llnl/build_src.py              | 20 +-------------------
 scripts/llnl/common_build_functions.py | 24 ++++++++++++++++++++++++
 scripts/llnl/run_benchmarks.py         |  9 +++------
 3 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/scripts/llnl/build_src.py b/scripts/llnl/build_src.py
index 158dd601c..6588cbcca 100755
--- a/scripts/llnl/build_src.py
+++ b/scripts/llnl/build_src.py
@@ -132,25 +132,7 @@ def main():
                 compiler = compiler.rsplit('-', 1)[0]
                 hostconfig = "%s-%s-%s.cmake" % (hostname, sys_type, compiler)
 
-            # First try with where uberenv generates host-configs.
-            hostconfig_path = os.path.join(repo_dir, hostconfig)
-            if not os.path.isfile(hostconfig_path):
-                print("[INFO: Looking for hostconfig at %s]" % hostconfig_path)
-                print("[WARNING: Spack generated host-config not found, trying with predefined]")
-
-                # Then look into project predefined host-configs.
-                hostconfig_path = os.path.join(repo_dir, "host-configs", hostconfig)
-                if not os.path.isfile(hostconfig_path):
-                    print("[INFO: Looking for hostconfig at %s]" % hostconfig_path)
-                    print("[WARNING: Predefined host-config not found, trying with Docker]")
-
-                    # Otherwise look into project predefined Docker host-configs.
-                    hostconfig_path = os.path.join(repo_dir, "host-configs", "docker", hostconfig)
-                    if not os.path.isfile(hostconfig_path):
-                        print("[INFO: Looking for hostconfig at %s]" % hostconfig_path)
-                        print("[WARNING: Predefined Docker host-config not found]")
-                        print("[ERROR: Could not find any host-configs in any known path. Try giving fully qualified path.]")
-                        return 1
+            hostconfig_path = get_host_config_path(repo_dir, hostconfig)
 
             test_root = get_build_and_test_root(repo_dir, timestamp)
             os.mkdir(test_root)
diff --git a/scripts/llnl/common_build_functions.py b/scripts/llnl/common_build_functions.py
index 927aacf17..35971b902 100755
--- a/scripts/llnl/common_build_functions.py
+++ b/scripts/llnl/common_build_functions.py
@@ -713,3 +713,27 @@ def convertSecondsToReadableTime(seconds):
     m, s = divmod(seconds, 60)
     h, m = divmod(m, 60)
     return "%d:%02d:%02d" % (h, m, s)
+
+
+def get_host_config_path(repo_dir, host_config):
+    # First try with where uberenv generates host-configs.
+    host_config_path = os.path.join(repo_dir, host_config)
+    if not os.path.isfile(host_config_path):
+        print("[INFO: Looking for host_config at %s]" % host_config_path)
+        print("[WARNING: Spack generated host-config not found, trying with predefined]")
+
+        # Then look into project predefined host-configs.
+        host_config_path = os.path.join(repo_dir, "host-configs", host_config)
+        if not os.path.isfile(host_config_path):
+            print("[INFO: Looking for host_config at %s]" % host_config_path)
+            print("[WARNING: Predefined host-config not found, trying with Docker]")
+
+            # Otherwise look into project predefined Docker host-configs.
+            host_config_path = os.path.join(repo_dir, "host-configs", "docker", host_config)
+            if not os.path.isfile(host_config_path):
+                print("[INFO: Looking for host_config at %s]" % host_config_path)
+                print("[WARNING: Predefined Docker host-config not found]")
+                print("[ERROR: Could not find any host-configs in any known path. Try giving fully qualified path.]")
+                sys.exit(1)
+
+    return host_config_path
diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py
index aef34bf62..3211eaa2e 100755
--- a/scripts/llnl/run_benchmarks.py
+++ b/scripts/llnl/run_benchmarks.py
@@ -31,7 +31,7 @@ def parse_args():
     parser.add_argument("-hc", "--host-config",
                     dest="host_config",
                     default=os.environ.get("HOST_CONFIG", None),
-                    help="Specific host-config file to build (defaults to HOST_CONFIG environment variable)")
+                    help="Specific host-config filename to build (defaults to HOST_CONFIG environment variable)")
     parser.add_argument("-sd", "--spot-directory",
                       dest="spot_dir",
                       default=get_shared_spot_dir(),
@@ -50,10 +50,6 @@ def parse_args():
         print("[ERROR: Both host_config argument and HOST_CONFIG environment variable unset!]")
         sys.exit(1)
 
-    if not os.path.exists(args["host_config"]):
-        print("[ERROR: Host config path does not exist: %s]" % args["host_config"])
-        sys.exit(1)
-
     return args
 
 
@@ -68,13 +64,14 @@ def main():
     # Vars
     repo_dir = get_repo_dir()
     test_root = get_build_and_test_root(repo_dir, timestamp)
+    host_config_path = get_host_config_path(repo_dir, host_config)
     host_config_root = get_host_config_root(host_config)
     benchmarks_output_file = os.path.join(test_root, "output.log.%s.benchmarks.txt" % host_config_root)
 
     # Build Serac
     os.chdir(repo_dir)
     os.makedirs(test_root, exist_ok=True)
-    build_and_test_host_config(test_root=test_root, host_config=host_config,
+    build_and_test_host_config(test_root=test_root, host_config=host_config_path,
                                report_to_stdout=True, extra_cmake_options=cmake_options,
                                skip_install=True, skip_tests=True)
 

From 7c95a15d784f38f9ade69b663a563e78877b88a7 Mon Sep 17 00:00:00 2001
From: chapman39 <chapman39@llnl.gov>
Date: Wed, 9 Oct 2024 13:38:33 -0700
Subject: [PATCH 04/19] Mention path to shared caliper files

---
 src/docs/sphinx/dev_guide/profiling.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst
index 48df8a818..274693971 100644
--- a/src/docs/sphinx/dev_guide/profiling.rst
+++ b/src/docs/sphinx/dev_guide/profiling.rst
@@ -136,6 +136,8 @@ files:
 - `SPOT CZ <https://lc.llnl.gov/spot2>`_
 - `SPOT RZ <https://rzlc.llnl.gov/spot2>`_
 
+The shared Caliper files for Serac are located here: https://lc.llnl.gov/spot2/?sf=/usr/WS2/smithdev/califiles
+
 .. note::
   There is a bug in SPOT where if you remove Caliper files from a directory, they still show up on SPOT - if you've
   visualized them previously. The current workaround is by removing the ``llnl.gov`` site cache manually.

From 1a1097b63a96fa52a98ab3bc3ff46d76f7ea5801 Mon Sep 17 00:00:00 2001
From: chapman39 <chapman39@llnl.gov>
Date: Wed, 9 Oct 2024 14:24:38 -0700
Subject: [PATCH 05/19] increase ci benchmark alloc time

---
 .gitlab/build_blueos.yml     | 2 +-
 .gitlab/build_toss4.yml      | 4 ++--
 .gitlab/build_toss4_cray.yml | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab/build_blueos.yml b/.gitlab/build_blueos.yml
index 2cb47e3ba..dc712dada 100644
--- a/.gitlab/build_blueos.yml
+++ b/.gitlab/build_blueos.yml
@@ -68,5 +68,5 @@ blueos-clang_10_0_1-benchmarks:
     COMPILER: "clang@10.0.1"
     HOST_CONFIG: "lassen-blueos_3_ppc64le_ib_p9-${COMPILER}_cuda.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "30"
+    ALLOC_TIME: "60"
   extends: [.benchmarks_build_on_blueos, .with_cuda]
diff --git a/.gitlab/build_toss4.yml b/.gitlab/build_toss4.yml
index 58d79ed54..e0771cd10 100644
--- a/.gitlab/build_toss4.yml
+++ b/.gitlab/build_toss4.yml
@@ -95,7 +95,7 @@ toss4-clang_14_0_6-benchmarks:
     COMPILER: "clang@14.0.6"
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "30"
+    ALLOC_TIME: "60"
   extends: .benchmarks_build_on_toss4
 
 toss4-gcc_10_3_1-benchmarks:
@@ -103,5 +103,5 @@ toss4-gcc_10_3_1-benchmarks:
     COMPILER: "gcc@10.3.1"
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "30"
+    ALLOC_TIME: "60"
   extends: .benchmarks_build_on_toss4
diff --git a/.gitlab/build_toss4_cray.yml b/.gitlab/build_toss4_cray.yml
index 17c0bf6c9..f106d2eab 100644
--- a/.gitlab/build_toss4_cray.yml
+++ b/.gitlab/build_toss4_cray.yml
@@ -54,5 +54,5 @@ toss4_cray-clang_17_0_0-benchmarks:
     COMPILER: "clang@17.0.0"
     HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "30"
+    ALLOC_TIME: "60"
   extends: .benchmarks_build_on_toss4_cray

From 0e932144f134b4d4ec269015c5ecf16cb8d1554a Mon Sep 17 00:00:00 2001
From: chapman39 <chapman39@llnl.gov>
Date: Wed, 9 Oct 2024 17:01:26 -0700
Subject: [PATCH 06/19] Check if petsc is enabled before running petsc
 multigrid

---
 .../benchmarks/physics_benchmark_solid_nonlinear_solve.cpp      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp
index 23e98abb6..40a5b763a 100644
--- a/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp
+++ b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp
@@ -357,9 +357,11 @@ int main(int argc, char* argv[])
     functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::MULTIGRID, 5e-10);
     SERAC_MARK_END("Multigrid Preconditioner");
 
+#ifdef SERAC_USE_PETSC
     SERAC_MARK_BEGIN("Petsc Multigrid Preconditioner");
     functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::PETSC_MULTIGRID, 5e-10);
     SERAC_MARK_END("Petsc Multigrid Preconditioner");
+#endif
   } else {
     SERAC_SET_METADATA("nonlinear solver", nonlinSolveToString(nonlinSolve));
     SERAC_SET_METADATA("preconditioner", precToString(prec));

From 8849962bcf3144305edaf84a6645b822d244d63b Mon Sep 17 00:00:00 2001
From: chapman39 <chapman39@llnl.gov>
Date: Wed, 9 Oct 2024 17:04:46 -0700
Subject: [PATCH 07/19] return result of benchmarks

---
 scripts/llnl/run_benchmarks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py
index 3211eaa2e..f60933b96 100755
--- a/scripts/llnl/run_benchmarks.py
+++ b/scripts/llnl/run_benchmarks.py
@@ -84,7 +84,7 @@ def main():
     os.chdir(build_dir)
 
     # Run benchmarks
-    shell_exec("make run_benchmarks", echo=True, print_output=True, output_file=benchmarks_output_file)
+    result = shell_exec("make run_benchmarks", echo=True, print_output=True, output_file=benchmarks_output_file)
 
     # Move resulting .cali files to specified directory
     os.makedirs(spot_dir, exist_ok=True)
@@ -99,7 +99,7 @@ def main():
     else:
         print("[View SPOT directory here: https://lc.llnl.gov/spot2/?sf={0}]".format(spot_dir))
 
-    return 0
+    return result; 
 
 
 if __name__ == "__main__":

From 77706dd2fefee266719e912c7461222c7e662225 Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Thu, 10 Oct 2024 09:51:33 -0700
Subject: [PATCH 08/19] increase deadline time to allow for longer jobs to stay
 on the queue longer

---
 .gitlab/build_toss4.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/build_toss4.yml b/.gitlab/build_toss4.yml
index e0771cd10..896813dc8 100644
--- a/.gitlab/build_toss4.yml
+++ b/.gitlab/build_toss4.yml
@@ -2,7 +2,7 @@
 # This is the shared configuration of jobs for toss4
 .on_toss4:
   variables:
-    SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+1hour -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}"
+    SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+90minutes -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}"
   tags:
     - batch
     - ruby

From 5d266efc6dca65651d4f736ff49476454a362ddf Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Fri, 25 Oct 2024 14:09:33 -0700
Subject: [PATCH 09/19] Ensure all cray jobs are disabled until all developers
 have access

---
 .gitlab-ci.yml               |  3 ++-
 .gitlab/build_toss4_cray.yml | 16 ++++++++--------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2ea9506aa..2f4fec344 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -90,6 +90,7 @@ variables:
 include:
   - local: .gitlab/build_blueos.yml
   - local: .gitlab/build_toss4.yml
-  - local: .gitlab/build_toss4_cray.yml
+  # Disabling cray until all developers have access to the tioga machine
+  # - local: .gitlab/build_toss4_cray.yml
   - project: 'lc-templates/id_tokens'
     file: 'id_tokens.yml'
diff --git a/.gitlab/build_toss4_cray.yml b/.gitlab/build_toss4_cray.yml
index cb6e12ff8..4fb53b6a7 100644
--- a/.gitlab/build_toss4_cray.yml
+++ b/.gitlab/build_toss4_cray.yml
@@ -34,14 +34,14 @@
 # Build jobs
 
 # Only run integration tests on one spec
-#toss4_cray-clang_17_0_0-src:
-#  variables:
-#    COMPILER: "clang@17.0.0"
-#    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
-#    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
-#    ALLOC_NODES: "1"
-#    ALLOC_TIME: "30"
-#  extends: .src_build_on_toss4_cray
+toss4_cray-clang_17_0_0-src:
+ variables:
+   COMPILER: "clang@17.0.0"
+   HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
+   EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
+   ALLOC_NODES: "1"
+   ALLOC_TIME: "30"
+ extends: .src_build_on_toss4_cray
 
 toss4_cray-clang_17_0_0-full:
   variables:

From d0fd0c3ac9bcad8147e68c6e0e9fc303e6027158 Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Fri, 25 Oct 2024 14:42:51 -0700
Subject: [PATCH 10/19] increase alloc time on benchmarks now that they take
 longer

---
 .gitlab/build_blueos.yml     | 2 +-
 .gitlab/build_toss4.yml      | 7 ++++---
 .gitlab/build_toss4_cray.yml | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.gitlab/build_blueos.yml b/.gitlab/build_blueos.yml
index dc712dada..bed3aa909 100644
--- a/.gitlab/build_blueos.yml
+++ b/.gitlab/build_blueos.yml
@@ -68,5 +68,5 @@ blueos-clang_10_0_1-benchmarks:
     COMPILER: "clang@10.0.1"
     HOST_CONFIG: "lassen-blueos_3_ppc64le_ib_p9-${COMPILER}_cuda.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "60"
+    ALLOC_TIME: "120"
   extends: [.benchmarks_build_on_blueos, .with_cuda]
diff --git a/.gitlab/build_toss4.yml b/.gitlab/build_toss4.yml
index 896813dc8..bfbf0cb58 100644
--- a/.gitlab/build_toss4.yml
+++ b/.gitlab/build_toss4.yml
@@ -2,7 +2,8 @@
 # This is the shared configuration of jobs for toss4
 .on_toss4:
   variables:
-    SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+90minutes -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}"
+    SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+${ALLOC_DEADLINE}minutes -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}"
+    ALLOC_DEADLINE: $((ALLOC_TIME+10))
   tags:
     - batch
     - ruby
@@ -95,7 +96,7 @@ toss4-clang_14_0_6-benchmarks:
     COMPILER: "clang@14.0.6"
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "60"
+    ALLOC_TIME: "120"
   extends: .benchmarks_build_on_toss4
 
 toss4-gcc_10_3_1-benchmarks:
@@ -103,5 +104,5 @@ toss4-gcc_10_3_1-benchmarks:
     COMPILER: "gcc@10.3.1"
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "60"
+    ALLOC_TIME: "120"
   extends: .benchmarks_build_on_toss4
diff --git a/.gitlab/build_toss4_cray.yml b/.gitlab/build_toss4_cray.yml
index 4fb53b6a7..56787c3b8 100644
--- a/.gitlab/build_toss4_cray.yml
+++ b/.gitlab/build_toss4_cray.yml
@@ -54,5 +54,5 @@ toss4_cray-clang_17_0_0-benchmarks:
     COMPILER: "clang@17.0.0"
     HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "60"
+    ALLOC_TIME: "120"
   extends: .benchmarks_build_on_toss4_cray

From cbcc38f5ebd5c0de65b33ef5ef0ee82b8430a546 Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Fri, 25 Oct 2024 15:53:07 -0700
Subject: [PATCH 11/19] add alloc_deadline variable to all toss4 jobs

---
 .gitlab/build_toss4.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.gitlab/build_toss4.yml b/.gitlab/build_toss4.yml
index bfbf0cb58..5ceeda03a 100644
--- a/.gitlab/build_toss4.yml
+++ b/.gitlab/build_toss4.yml
@@ -3,7 +3,6 @@
 .on_toss4:
   variables:
     SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+${ALLOC_DEADLINE}minutes -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}"
-    ALLOC_DEADLINE: $((ALLOC_TIME+10))
   tags:
     - batch
     - ruby
@@ -46,6 +45,7 @@ toss4-clang_14_0_6-src:
     DO_INTEGRATION_TESTS: "yes"
     ALLOC_NODES: "2"
     ALLOC_TIME: "30"
+    ALLOC_DEADLINE: "60"
   extends: .src_build_on_toss4
 
 toss4-gcc_10_3_1-src:
@@ -55,6 +55,7 @@ toss4-gcc_10_3_1-src:
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
+    ALLOC_DEADLINE: "60"
   extends: .src_build_on_toss4
 
 toss4-gcc_10_3_1-src-no-tribol:
@@ -64,6 +65,7 @@ toss4-gcc_10_3_1-src-no-tribol:
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -UTRIBOL_DIR"
     ALLOC_NODES: "1"
     ALLOC_TIME: "30"
+    ALLOC_DEADLINE: "60"
   extends: .src_build_on_toss4
 
 toss4-gcc_10_3_1-src-no-optional-solvers:
@@ -73,6 +75,7 @@ toss4-gcc_10_3_1-src-no-optional-solvers:
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -USUNDIALS_DIR -UPETSC_DIR"
     ALLOC_NODES: "1"
     ALLOC_TIME: "20"
+    ALLOC_DEADLINE: "60"
   extends: .src_build_on_toss4
 
 toss4-clang_14_0_6-full:
@@ -81,6 +84,7 @@ toss4-clang_14_0_6-full:
     SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
+    ALLOC_DEADLINE: "60"
   extends: .full_build_on_toss4
 
 toss4-gcc_10_3_1-full:
@@ -89,6 +93,7 @@ toss4-gcc_10_3_1-full:
     SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
+    ALLOC_DEADLINE: "60"
   extends: .full_build_on_toss4
 
 toss4-clang_14_0_6-benchmarks:
@@ -97,6 +102,7 @@ toss4-clang_14_0_6-benchmarks:
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
     ALLOC_TIME: "120"
+    ALLOC_DEADLINE: "180"
   extends: .benchmarks_build_on_toss4
 
 toss4-gcc_10_3_1-benchmarks:
@@ -105,4 +111,5 @@ toss4-gcc_10_3_1-benchmarks:
     HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
     ALLOC_NODES: "1"
     ALLOC_TIME: "120"
+    ALLOC_DEADLINE: "180"
   extends: .benchmarks_build_on_toss4

From f337dde94ec30252f1d1d061274c0e30c6ae49ec Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Mon, 28 Oct 2024 14:21:48 -0700
Subject: [PATCH 12/19] build benchmarks as release

---
 scripts/llnl/run_benchmarks.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py
index f60933b96..9ac79d580 100755
--- a/scripts/llnl/run_benchmarks.py
+++ b/scripts/llnl/run_benchmarks.py
@@ -27,7 +27,8 @@ def parse_args():
     parser.add_argument("-e", "--extra-cmake-options",
                       dest="extra_cmake_options",
                       default=os.environ.get("EXTRA_CMAKE_OPTIONS", ""),
-                      help="Extra cmake options to add to the cmake configure line. Note '-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF' is always appended.")
+                      help="Extra cmake options to add to the cmake configure line. Note that options to enable benchmarks, " +
+                           "disable docs, and build as Release are always appended.")
     parser.add_argument("-hc", "--host-config",
                     dest="host_config",
                     default=os.environ.get("HOST_CONFIG", None),
@@ -56,7 +57,7 @@ def parse_args():
 def main():
     # Args
     args = parse_args()
-    cmake_options = args["extra_cmake_options"] + " -DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF"
+    cmake_options = args["extra_cmake_options"] + " -DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Release"
     host_config = args["host_config"]
     spot_dir = args["spot_dir"]
     timestamp = args["timestamp"]

From faf3ff9a64df16d62d4327c86777322224bcc152 Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Mon, 28 Oct 2024 17:03:44 -0700
Subject: [PATCH 13/19] increase benchmark ctests to be 1.5 hours per job

---
 src/serac/physics/benchmarks/CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt
index 137a89978..97adfdb7a 100644
--- a/src/serac/physics/benchmarks/CMakeLists.txt
+++ b/src/serac/physics/benchmarks/CMakeLists.txt
@@ -21,6 +21,9 @@ foreach(physics_benchmark ${physics_benchmark_targets})
                        FOLDER serac/benchmarks
                        )
 
+    # Increase benchmark time limit to one hour
+    set_tests_properties(${physics_benchmark} PROPERTIES TIMEOUT 4800)
+
     # Add benchmarks with various task counts
     foreach(task_count 1 4 16)
         blt_add_benchmark(NAME ${physics_benchmark}_${task_count}_task_count

From 2ba2921d22ee554bd356615eebbcb725bfd052b1 Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Mon, 28 Oct 2024 17:11:22 -0700
Subject: [PATCH 14/19] comment

---
 src/serac/physics/benchmarks/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt
index 97adfdb7a..c0b5db041 100644
--- a/src/serac/physics/benchmarks/CMakeLists.txt
+++ b/src/serac/physics/benchmarks/CMakeLists.txt
@@ -21,7 +21,7 @@ foreach(physics_benchmark ${physics_benchmark_targets})
                        FOLDER serac/benchmarks
                        )
 
-    # Increase benchmark time limit to one hour
+    # Increase benchmark time limit
     set_tests_properties(${physics_benchmark} PROPERTIES TIMEOUT 4800)
 
     # Add benchmarks with various task counts

From 9c1b3e44d30791ff23255f239876225245cc1e6a Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Tue, 29 Oct 2024 14:54:27 -0700
Subject: [PATCH 15/19] set property to actual test not exec

---
 src/serac/physics/benchmarks/CMakeLists.txt | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt
index c0b5db041..9c9c94982 100644
--- a/src/serac/physics/benchmarks/CMakeLists.txt
+++ b/src/serac/physics/benchmarks/CMakeLists.txt
@@ -13,23 +13,24 @@ set(physics_benchmark_targets
     )
 
 # Create executable for each benchmark
-foreach(physics_benchmark ${physics_benchmark_targets})
-    blt_add_executable(NAME ${physics_benchmark}
-                       SOURCES ${physics_benchmark}.cpp
+foreach(physics_benchmark_exec ${physics_benchmark_targets})
+    blt_add_executable(NAME ${physics_benchmark_exec}
+                       SOURCES ${physics_benchmark_exec}.cpp
                        DEPENDS_ON ${physics_benchmark_depends}
                        OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks
                        FOLDER serac/benchmarks
                        )
 
-    # Increase benchmark time limit
-    set_tests_properties(${physics_benchmark} PROPERTIES TIMEOUT 4800)
-
     # Add benchmarks with various task counts
     foreach(task_count 1 4 16)
-        blt_add_benchmark(NAME ${physics_benchmark}_${task_count}_task_count
-                          COMMAND ${physics_benchmark}
+        set(physics_benchmark ${physics_benchmark_exec}_${task_count}_task_count)
+        blt_add_benchmark(NAME ${physics_benchmark}
+                          COMMAND ${physics_benchmark_exec}
                           NUM_MPI_TASKS ${task_count}
                           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
                           )
+
+        # Increase benchmark time limit
+        set_tests_properties(${physics_benchmark} PROPERTIES TIMEOUT 4800)
     endforeach()
 endforeach()

From 65a276993ecc2540bc0711cf3c4598ecf8287a9d Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Mon, 4 Nov 2024 09:37:31 -0800
Subject: [PATCH 16/19] use single variable to switch between ci pipelines

---
 .gitlab-ci.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2f4fec344..73abb4545 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,17 +17,24 @@ variables:
 .run_update_uberenv: &run_update_uberenv |
   [[ -n "${UPDATE_UBERENV}" ]] && ./scripts/gitlab/update-uberenv.sh "${UPDATE_UBERENV}"
 
+# Run src build each time a merge request has been updated
 .src_workflow:
   rules:
-    - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD != "ON"'
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result"
+    - if: $CI_PIPELINE_SOURCE == "push"
+      when: never
+    - if: $CI_COMMIT_BRANCH == "develop"
+      when: never
 
+# Run full build as a nightly scheduled pipeline
 .full_workflow:
   rules:
-    - if: '$FULL_BUILD == "ON" && $BENCHMARKS_BUILD != "ON"'
+    - if: $CI_WORKFLOW_TYPE == "full"
 
+# Run benchmarks build as a weekly scheduled pipeline
 .benchmarks_workflow:
   rules:
-    - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD == "ON"'
+    - if: $CI_WORKFLOW_TYPE == "benchmarks"
 
 ####
 # Templates

From 2643068c545f58c3ea933f7ef058fe87b1b645ac Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Mon, 4 Nov 2024 09:47:28 -0800
Subject: [PATCH 17/19] set build_dir in a more straightforward way

---
 scripts/llnl/run_benchmarks.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py
index 9ac79d580..d2d388480 100755
--- a/scripts/llnl/run_benchmarks.py
+++ b/scripts/llnl/run_benchmarks.py
@@ -68,6 +68,7 @@ def main():
     host_config_path = get_host_config_path(repo_dir, host_config)
     host_config_root = get_host_config_root(host_config)
     benchmarks_output_file = os.path.join(test_root, "output.log.%s.benchmarks.txt" % host_config_root)
+    build_dir = os.path.join(test_root, "build-%s" % host_config_root)
 
     # Build Serac
     os.chdir(repo_dir)
@@ -77,11 +78,6 @@ def main():
                                skip_install=True, skip_tests=True)
 
     # Go to build location
-    build_dir=""
-    dirs = glob.glob(pjoin(test_root, "*"))
-    for dir in dirs:
-        if os.path.exists(dir) and "build-" in dir:
-            build_dir=dir
     os.chdir(build_dir)
 
     # Run benchmarks

From 6a0efc16229f69a919c7a008e37d0764ebb39a48 Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Mon, 4 Nov 2024 09:49:06 -0800
Subject: [PATCH 18/19] attempt to improve variable names

---
 src/serac/physics/benchmarks/CMakeLists.txt | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt
index 9c9c94982..846d35f9e 100644
--- a/src/serac/physics/benchmarks/CMakeLists.txt
+++ b/src/serac/physics/benchmarks/CMakeLists.txt
@@ -6,14 +6,14 @@
 
 set(physics_benchmark_depends serac_physics)
 
-set(physics_benchmark_targets
+set(physics_benchmarks
     physics_benchmark_functional
     physics_benchmark_solid_nonlinear_solve
     physics_benchmark_thermal
     )
 
 # Create executable for each benchmark
-foreach(physics_benchmark_exec ${physics_benchmark_targets})
+foreach(physics_benchmark_exec ${physics_benchmarks})
     blt_add_executable(NAME ${physics_benchmark_exec}
                        SOURCES ${physics_benchmark_exec}.cpp
                        DEPENDS_ON ${physics_benchmark_depends}
@@ -23,14 +23,14 @@ foreach(physics_benchmark_exec ${physics_benchmark_targets})
 
     # Add benchmarks with various task counts
     foreach(task_count 1 4 16)
-        set(physics_benchmark ${physics_benchmark_exec}_${task_count}_task_count)
-        blt_add_benchmark(NAME ${physics_benchmark}
+        set(physics_benchmark_name ${physics_benchmark_exec}_${task_count}_task_count)
+        blt_add_benchmark(NAME ${physics_benchmark_name}
                           COMMAND ${physics_benchmark_exec}
                           NUM_MPI_TASKS ${task_count}
                           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
                           )
 
         # Increase benchmark time limit
-        set_tests_properties(${physics_benchmark} PROPERTIES TIMEOUT 4800)
+        set_tests_properties(${physics_benchmark_name} PROPERTIES TIMEOUT 4800)
     endforeach()
 endforeach()

From 53cdc0c5b85cce1806b4d31b1979fe77a6c5481f Mon Sep 17 00:00:00 2001
From: Alex Tyler Chapman <chapman39@llnl.gov>
Date: Mon, 4 Nov 2024 11:38:58 -0800
Subject: [PATCH 19/19] fix on-push ci

---
 .gitlab-ci.yml | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 73abb4545..72ff8e3a8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,24 +17,20 @@ variables:
 .run_update_uberenv: &run_update_uberenv |
   [[ -n "${UPDATE_UBERENV}" ]] && ./scripts/gitlab/update-uberenv.sh "${UPDATE_UBERENV}"
 
-# Run src build each time a merge request has been updated
+# Run src build each push
 .src_workflow:
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result"
-    - if: $CI_PIPELINE_SOURCE == "push"
-      when: never
-    - if: $CI_COMMIT_BRANCH == "develop"
-      when: never
+    - if: $SERAC_CI_WORKFLOW_TYPE != "full" && $SERAC_CI_WORKFLOW_TYPE != "benchmarks"
 
 # Run full build as a nightly scheduled pipeline
 .full_workflow:
   rules:
-    - if: $CI_WORKFLOW_TYPE == "full"
+    - if: $SERAC_CI_WORKFLOW_TYPE == "full"
 
 # Run benchmarks build as a weekly scheduled pipeline
 .benchmarks_workflow:
   rules:
-    - if: $CI_WORKFLOW_TYPE == "benchmarks"
+    - if: $SERAC_CI_WORKFLOW_TYPE == "benchmarks"
 
 ####
 # Templates