From 1d3c9b5c253c3cf0244da24855e3a5aeb58001f1 Mon Sep 17 00:00:00 2001
From: Bob Yantosca <yantosca@seas.harvard.edu>
Date: Tue, 1 Aug 2023 17:32:13 -0400
Subject: [PATCH 1/6] Add n_cores to the benchmark YAML files

benchmark/cloud/template.1hr_benchmark.yml
benchmark/cloud/template.1mo_benchmark.yml
benchmark/config/1mo_benchmark.yml
benchmark/config/1yr_ch4_benchmark.yml
benchmark/config/1yr_fullchem_benchmark.yml
benchmark/config/1yr_tt_benchmark.yml
- Update comments
- Add n_cores: -1 which will by default turn on parallelization
  and use as many cores as possible.

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>
---
 benchmark/cloud/template.1hr_benchmark.yml  | 31 +++++++++++++-----
 benchmark/cloud/template.1mo_benchmark.yml  | 31 +++++++++++++-----
 benchmark/config/1mo_benchmark.yml          | 35 +++++++++++++++------
 benchmark/config/1yr_ch4_benchmark.yml      | 29 +++++++++++++----
 benchmark/config/1yr_fullchem_benchmark.yml | 23 ++++++++++++--
 benchmark/config/1yr_tt_benchmark.yml       | 33 ++++++++++++++-----
 6 files changed, 142 insertions(+), 40 deletions(-)

diff --git a/benchmark/cloud/template.1hr_benchmark.yml b/benchmark/cloud/template.1hr_benchmark.yml
index b9f88942..672ba317 100644
--- a/benchmark/cloud/template.1hr_benchmark.yml
+++ b/benchmark/cloud/template.1hr_benchmark.yml
@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: False
       resolution: c24
 #
-# options: Specify the types of comparisons to perform
+# options: Customizes the benchmark plot output.
 #
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: FullChemBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc:
       run: False
@@ -92,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: False
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: False
     plot_emis: False
@@ -110,3 +120,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use as many cores as possible
+  #  N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1
diff --git a/benchmark/cloud/template.1mo_benchmark.yml b/benchmark/cloud/template.1mo_benchmark.yml
index 2f1ad869..2e149efd 100644
--- a/benchmark/cloud/template.1mo_benchmark.yml
+++ b/benchmark/cloud/template.1mo_benchmark.yml
@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: False
       resolution: c24
 #
-# options: Specify the types of comparisons to perform
+# options: Customizes the benchmark plot output.
 #
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: FullChemBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc:
       run: False
@@ -92,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: False
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -110,3 +120,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use as many cores as possible
+  #  N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1
diff --git a/benchmark/config/1mo_benchmark.yml b/benchmark/config/1mo_benchmark.yml
index 21ea61df..594a3e86 100644
--- a/benchmark/config/1mo_benchmark.yml
+++ b/benchmark/config/1mo_benchmark.yml
@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,13 +76,19 @@ data:
       is_pre_14.0: False
       resolution: c24
 #
-# options: Specify the types of comparisons to perform
-#   
+# options: Customizes the benchmark plot output.
+#
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: FullChemBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc: 
-      run: True # True to run this comparison
+      run: True
       dir: GCC_version_comparison
       tables_subdir: Tables
     gchp_vs_gcc: 
@@ -92,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs: 
       run: True
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-# 
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -110,3 +120,10 @@ options:
     plot_options: # Plot concentrations and emissions by category?
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use as many cores as possible
+  #  N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1
diff --git a/benchmark/config/1yr_ch4_benchmark.yml b/benchmark/config/1yr_ch4_benchmark.yml
index 80452841..bf35aeb7 100644
--- a/benchmark/config/1yr_ch4_benchmark.yml
+++ b/benchmark/config/1yr_ch4_benchmark.yml
@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
-# (4) If necessary, edit labels for the dev and ref versions
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and tables,
+#     as well as other plotting options.
+# (4) If necessary, edit labels for the dev and ref versions.
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: False              # for gcpy_test_data, edit if needed
       resolution: c24                 # for gcpy_test_data, edit if needed   
 #
-# options: Specify the types of comparisons to perform
+# options: Customize the benchmark plot output.
 # 
 options:
+#
+# bmk_type: Specifies the type of benchmark.
+#
   bmk_type: CH4Benchmark
+#
+# comparisons: Specifies the comparisons to perform.
+#
   comparisons:
     gcc_vs_gcc: 
       run: True # True to run this comparison
@@ -93,7 +103,7 @@ options:
       run: False
       dir: GCHP_GCC_diff_of_diffs
 #
-# outputs: Types of output to generate (plots/tables)
+# outputs: Specifies the plots and tables to generate.
 #
   outputs:
     plot_conc: True
@@ -110,3 +120,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+#
+# n_cores: Specify the number of cores to use.
+# -1: Use as many cores as possible
+#  N: Use N cores
+#  1: Disable parallelization (use a single core)
+#
+  n_cores: -1
diff --git a/benchmark/config/1yr_fullchem_benchmark.yml b/benchmark/config/1yr_fullchem_benchmark.yml
index 3cfe97dc..2b62e8f3 100644
--- a/benchmark/config/1yr_fullchem_benchmark.yml
+++ b/benchmark/config/1yr_fullchem_benchmark.yml
@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data
 # (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -74,10 +78,16 @@ data:
       is_pre_14.0: False              # for gcpy_test_data, edit if needed
       resolution: c24                 # for gcpy_test_data, edit if needed   
 #
-# options: Specify the types of comparisons to perform
+# options: Customize the benchmark plot output.
 # 
 options:
+#
+# bmk_type: Specifies the type of benchmark
+#
   bmk_type: FullChemBenchmark
+#
+# comparisons: Specifies the comparisons to perform.
+#
   comparisons:
     gcc_vs_gcc: 
       run: True # True to run this comparison
@@ -113,3 +123,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+#
+# n_cores: Specify the number of cores to use:
+# -1: Use as many cores as possible
+#  N: Use N cores
+#  1: Disable parallelization (use a single core)
+#
+  n_cores: -1
diff --git a/benchmark/config/1yr_tt_benchmark.yml b/benchmark/config/1yr_tt_benchmark.yml
index adca22f3..916b01cb 100644
--- a/benchmark/config/1yr_tt_benchmark.yml
+++ b/benchmark/config/1yr_tt_benchmark.yml
@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
-# (4) If necessary, edit labels for the dev and ref versions
+#
+# (1) Edit the path variables so that they point to folders w/
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
+# (4) If necessary, edit labels for the dev and ref versions.
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: True               # for gcpy_test_data, edit if needed
       resolution: c48                 # for gcpy_test_data, edit if needed
 #
-# options: Specify the types of comparisons to perform
+# options: Customize the benchmark plot output.
 #
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: TransportTracersBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc:
       run: True
@@ -90,9 +100,9 @@ options:
       dir: GCHP_version_comparison
       tables_subdir: Tables
     # GCHP vs GCC diff of diffs not included in 1-yr tt benchmark
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_wetdep: True
@@ -101,3 +111,10 @@ options:
     mass_table: True
     ste_table: True
     cons_table: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use as many cores as possible
+  #  N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1

From 38db42b77436333dd42a4b35c1614d473d14e2da Mon Sep 17 00:00:00 2001
From: Bob Yantosca <yantosca@seas.harvard.edu>
Date: Wed, 2 Aug 2023 11:40:05 -0400
Subject: [PATCH 2/6] Now pass n_job to all of the benchmark plot routines

benchmark_modules/run_1yr_fullchem_benchmark.py
benchmark_modules/run_1yr_tt_benchmark.py
benchmark/run.benchmark.py
- Read the number of cores to use in plot jobs from the YAML config
  files into the config["options"]["n_cores"] dict entry.
- Pass "n_job=config["options"]["n_cores"] into plotting routines
  so that the number of cores to use in parallel can be specified
  (or to disable parallelization completely)

CHANGELOG.md
- Updated accordingly

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>
---
 CHANGELOG.md                                  |  2 ++
 .../modules/run_1yr_fullchem_benchmark.py     | 27 +++++++++++++++++++
 benchmark/modules/run_1yr_tt_benchmark.py     | 12 +++++++++
 benchmark/run_benchmark.py                    | 16 ++++++++++-
 4 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5b7d8cc7..191da666 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Added new routine `verify_variable_type` function in `gcpy/util.py`
 - Added new routine `format_number_for_table` in `util.py`
 - Added BrSALA and BrSALC to `emission_species.yml`
+- Added `options:n_cores` to all benchmark YAML config files
 
 ### Changed
 - Simplified the Github issues templates into two options: `new-feature-or-discussion.md` and `question-issue.md`
@@ -40,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Truncate names in benchmark & emissions tables to improve readability
 - Add TransportTracers species names to `gcpy/emissions_*.yml` files
 - Updated `docs/environment_files/environment.yml` to install `pyproj==3.6.0` via pip
+- Now pass `n_job=config["options"]["n_cores"]` to benchmark plotting routines
 
 ### Fixed
 - Generalized test for GCHP or GCClassic restart file in `regrid_restart_file.py`
diff --git a/benchmark/modules/run_1yr_fullchem_benchmark.py b/benchmark/modules/run_1yr_fullchem_benchmark.py
index 06d68eb2..607e270e 100755
--- a/benchmark/modules/run_1yr_fullchem_benchmark.py
+++ b/benchmark/modules/run_1yr_fullchem_benchmark.py
@@ -402,6 +402,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                 benchmark_type=bmk_type,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -427,6 +428,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                         "plot_options"]["by_spc_cat"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -469,6 +471,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                 benchmark_type=bmk_type,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -494,6 +497,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                     benchmark_type=bmk_type,
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -526,6 +530,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                 dev_interval=sec_per_month_dev,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
         # ==================================================================
@@ -563,6 +568,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                 weightsdir=config["paths"]["weights_dir"],
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -583,6 +589,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                     weightsdir=config["paths"]["weights_dir"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -620,6 +627,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                 weightsdir=config["paths"]["weights_dir"],
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -640,6 +648,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                     weightsdir=config["paths"]["weights_dir"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -923,6 +932,7 @@ def gcc_vs_gcc_ops_budg(mon):
                 benchmark_type=bmk_type,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -948,6 +958,7 @@ def gcc_vs_gcc_ops_budg(mon):
                         "plot_options"]["by_spc_cat"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==============================================================
@@ -991,6 +1002,7 @@ def gcc_vs_gcc_ops_budg(mon):
                 benchmark_type=bmk_type,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -1016,6 +1028,7 @@ def gcc_vs_gcc_ops_budg(mon):
                     benchmark_type=bmk_type,
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -1088,6 +1101,7 @@ def gcc_vs_gcc_ops_budg(mon):
                 weightsdir=config["paths"]["weights_dir"],
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -1108,6 +1122,7 @@ def gcc_vs_gcc_ops_budg(mon):
                     weightsdir=config["paths"]["weights_dir"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -1146,6 +1161,7 @@ def gcc_vs_gcc_ops_budg(mon):
                 weightsdir=config["paths"]["weights_dir"],
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -1166,6 +1182,7 @@ def gcc_vs_gcc_ops_budg(mon):
                     weightsdir=config["paths"]["weights_dir"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -1483,6 +1500,7 @@ def gchp_vs_gcc_ops_budg(mon):
                     "plot_options"]["by_spc_cat"],
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -1509,6 +1527,7 @@ def gchp_vs_gcc_ops_budg(mon):
                         "plot_options"]["by_spc_cat"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -1554,6 +1573,7 @@ def gchp_vs_gcc_ops_budg(mon):
                 benchmark_type=bmk_type,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -1580,6 +1600,7 @@ def gchp_vs_gcc_ops_budg(mon):
                     benchmark_type=bmk_type,
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -1656,6 +1677,7 @@ def gchp_vs_gcc_ops_budg(mon):
                 weightsdir=config["paths"]["weights_dir"],
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -1677,6 +1699,7 @@ def gchp_vs_gcc_ops_budg(mon):
                     weightsdir=config["paths"]["weights_dir"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -1717,6 +1740,7 @@ def gchp_vs_gcc_ops_budg(mon):
                 weightsdir=config["paths"]["weights_dir"],
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -1738,6 +1762,7 @@ def gchp_vs_gcc_ops_budg(mon):
                     weightsdir=config["paths"]["weights_dir"],
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -2050,6 +2075,7 @@ def gchp_vs_gchp_ops_budg(mon):
                 second_dev=gchp_dev,
                 cats_in_ugm3=None,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -2077,6 +2103,7 @@ def gchp_vs_gchp_ops_budg(mon):
                     second_dev=gchp_dev[mon_ind],
                     cats_in_ugm3=None,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
     # ==================================================================
diff --git a/benchmark/modules/run_1yr_tt_benchmark.py b/benchmark/modules/run_1yr_tt_benchmark.py
index 63b18a9a..231f9a10 100755
--- a/benchmark/modules/run_1yr_tt_benchmark.py
+++ b/benchmark/modules/run_1yr_tt_benchmark.py
@@ -385,6 +385,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                 restrict_cats=restrict_cats,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -407,6 +408,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                     restrict_cats=restrict_cats,
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -453,6 +455,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                     benchmark_type=bmk_type,
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
                 # ----------------------------------------------------------
@@ -474,6 +477,7 @@ def run_benchmark(config, bmk_year_ref, bmk_year_dev):
                         benchmark_type=bmk_type,
                         overwrite=True,
                         spcdb_dir=spcdb_dir,
+                        n_job=config["options"]["n_cores"]
                     )
 
 
@@ -666,6 +670,7 @@ def gcc_vs_gcc_mass_table(mon):
                 restrict_cats=restrict_cats,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -688,6 +693,7 @@ def gcc_vs_gcc_mass_table(mon):
                     restrict_cats=restrict_cats,
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -735,6 +741,7 @@ def gcc_vs_gcc_mass_table(mon):
                     benchmark_type=bmk_type,
                     normalize_by_area=True,
                     spcdb_dir=spcdb_dir,
+                    n_job=config["options"]["n_cores"]
                 )
 
                 # ----------------------------------------------------------
@@ -757,6 +764,7 @@ def gcc_vs_gcc_mass_table(mon):
                         benchmark_type=bmk_type,
                         normalize_by_area=True,
                         spcdb_dir=spcdb_dir,
+                        n_job=config["options"]["n_cores"]
                     )
 
         # ==================================================================
@@ -928,6 +936,7 @@ def gchp_vs_gcc_mass_table(mon):
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
                 cmpres=cmpres
+                n_job=config["options"]["n_cores"]
             )
 
             # --------------------------------------------------------------
@@ -951,6 +960,7 @@ def gchp_vs_gcc_mass_table(mon):
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
                     cmpres=cmpres
+                    n_job=config["options"]["n_cores"]
                 )
 
         # ==================================================================
@@ -1001,6 +1011,7 @@ def gchp_vs_gcc_mass_table(mon):
                     normalize_by_area=True,
                     spcdb_dir=spcdb_dir,
                     cmpres=cmpres
+                    n_job=config["options"]["n_cores"]
                 )
 
                 # ----------------------------------------------------------
@@ -1024,6 +1035,7 @@ def gchp_vs_gcc_mass_table(mon):
                         normalize_by_area=True,
                         spcdb_dir=spcdb_dir,
                         cmpres=cmpres
+                        n_job=config["options"]["n_cores"]
                     )
 
         # ==================================================================
diff --git a/benchmark/run_benchmark.py b/benchmark/run_benchmark.py
index 79098f30..f5af05be 100755
--- a/benchmark/run_benchmark.py
+++ b/benchmark/run_benchmark.py
@@ -428,6 +428,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gcc_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
         # ==================================================================
@@ -456,6 +457,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gcc_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
         # ==================================================================
@@ -503,6 +505,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gcc_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
         # ==================================================================
@@ -526,6 +529,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gcc_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
         # ==================================================================
@@ -774,6 +778,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]
             )
 
         # ==================================================================
@@ -809,6 +814,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -839,6 +845,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 devmet=devmet,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -867,6 +874,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -895,6 +903,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gcc_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -1001,7 +1010,7 @@ def run_benchmark_default(config):
                 is_gchp=True
             )
 
-            # Create plots
+            # Create table
             bmk.make_benchmark_operations_budget(
                 config["data"]["dev"]["gcc"]["version"],
                 ref,
@@ -1178,6 +1187,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gchp_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -1216,6 +1226,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gchp_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -1285,6 +1296,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gchp_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -1318,6 +1330,7 @@ def run_benchmark_default(config):
                 overwrite=True,
                 sigdiff_files=gchp_vs_gchp_sigdiff,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
         # ==================================================================
@@ -1568,6 +1581,7 @@ def run_benchmark_default(config):
                 second_dev=gchp_dev,
                 cats_in_ugm3=None,
                 spcdb_dir=spcdb_dir,
+                n_job=config["options"]["n_cores"]                
             )
 
 

From af27630f253519f562db4b8b998b7ebe0b958310 Mon Sep 17 00:00:00 2001
From: Bob Yantosca <yantosca@seas.harvard.edu>
Date: Fri, 4 Aug 2023 12:36:52 -0400
Subject: [PATCH 3/6] Add misisng comments in run_1yr_tt_benchmark.py

benchmark/modules/run_1yr_tt_benchmark.py:
- Added missing comments after the cmpres argument in a few calls
  to plotting code.

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>
---
 benchmark/modules/run_1yr_tt_benchmark.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/benchmark/modules/run_1yr_tt_benchmark.py b/benchmark/modules/run_1yr_tt_benchmark.py
index 231f9a10..cf371bad 100755
--- a/benchmark/modules/run_1yr_tt_benchmark.py
+++ b/benchmark/modules/run_1yr_tt_benchmark.py
@@ -935,7 +935,7 @@ def gchp_vs_gcc_mass_table(mon):
                 restrict_cats=restrict_cats,
                 overwrite=True,
                 spcdb_dir=spcdb_dir,
-                cmpres=cmpres
+                cmpres=cmpres,
                 n_job=config["options"]["n_cores"]
             )
 
@@ -959,7 +959,7 @@ def gchp_vs_gcc_mass_table(mon):
                     restrict_cats=restrict_cats,
                     overwrite=True,
                     spcdb_dir=spcdb_dir,
-                    cmpres=cmpres
+                    cmpres=cmpres,
                     n_job=config["options"]["n_cores"]
                 )
 
@@ -1010,7 +1010,7 @@ def gchp_vs_gcc_mass_table(mon):
                     benchmark_type=bmk_type,
                     normalize_by_area=True,
                     spcdb_dir=spcdb_dir,
-                    cmpres=cmpres
+                    cmpres=cmpres,
                     n_job=config["options"]["n_cores"]
                 )
 
@@ -1034,7 +1034,7 @@ def gchp_vs_gcc_mass_table(mon):
                         benchmark_type=bmk_type,
                         normalize_by_area=True,
                         spcdb_dir=spcdb_dir,
-                        cmpres=cmpres
+                        cmpres=cmpres,
                         n_job=config["options"]["n_cores"]
                     )
 

From abddd5b958c01c4173894a9baa46cb69f59ef171 Mon Sep 17 00:00:00 2001
From: Bob Yantosca <yantosca@seas.harvard.edu>
Date: Fri, 4 Aug 2023 13:54:44 -0400
Subject: [PATCH 4/6] Fix incorrect comments in *_benchmark.yml files

benchmark/cloud/template.1hr_benchmark.yml
benchmark/cloud/template.1mo_benchmark.yml
benchmark/config/1mo_benchmark.yml
benchmark/config/1yr_ch4_benchmark.yml
benchmark/config/1yr_fullchem_benchmark.yml
benchmark/config/1yr_tt_benchmark.yml
- Fix comments, now state that n_cores = -1 will use all available cores,
  ncores = -N will use N cores (it must be negative), and ncores=1
  will disable parallelization
- Indent comments to line up with YAML tags where necessary

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>
---
 benchmark/cloud/template.1hr_benchmark.yml  |  4 +--
 benchmark/cloud/template.1mo_benchmark.yml  |  4 +--
 benchmark/config/1mo_benchmark.yml          |  4 +--
 benchmark/config/1yr_ch4_benchmark.yml      | 30 ++++++++++-----------
 benchmark/config/1yr_fullchem_benchmark.yml | 30 ++++++++++-----------
 benchmark/config/1yr_tt_benchmark.yml       |  4 +--
 6 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/benchmark/cloud/template.1hr_benchmark.yml b/benchmark/cloud/template.1hr_benchmark.yml
index 672ba317..68006695 100644
--- a/benchmark/cloud/template.1hr_benchmark.yml
+++ b/benchmark/cloud/template.1hr_benchmark.yml
@@ -122,8 +122,8 @@ options:
       by_hco_cat: True
   #
   # n_cores: Specify the number of cores to use:
-  # -1: Use as many cores as possible
-  #  N: Use N cores
+  # -1: Use all available cores
+  # -N: Use N cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1
diff --git a/benchmark/cloud/template.1mo_benchmark.yml b/benchmark/cloud/template.1mo_benchmark.yml
index 2e149efd..7b9c65e6 100644
--- a/benchmark/cloud/template.1mo_benchmark.yml
+++ b/benchmark/cloud/template.1mo_benchmark.yml
@@ -122,8 +122,8 @@ options:
       by_hco_cat: True
   #
   # n_cores: Specify the number of cores to use:
-  # -1: Use as many cores as possible
-  #  N: Use N cores
+  # -1: Use all available cores
+  # -N: Use N cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1
diff --git a/benchmark/config/1mo_benchmark.yml b/benchmark/config/1mo_benchmark.yml
index 594a3e86..5a122d40 100644
--- a/benchmark/config/1mo_benchmark.yml
+++ b/benchmark/config/1mo_benchmark.yml
@@ -122,8 +122,8 @@ options:
       by_hco_cat: True
   #
   # n_cores: Specify the number of cores to use:
-  # -1: Use as many cores as possible
-  #  N: Use N cores
+  # -1: Use all avaiable cores
+  # -N: Use N cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1
diff --git a/benchmark/config/1yr_ch4_benchmark.yml b/benchmark/config/1yr_ch4_benchmark.yml
index bf35aeb7..2a656f69 100644
--- a/benchmark/config/1yr_ch4_benchmark.yml
+++ b/benchmark/config/1yr_ch4_benchmark.yml
@@ -79,13 +79,13 @@ data:
 # options: Customize the benchmark plot output.
 # 
 options:
-#
-# bmk_type: Specifies the type of benchmark.
-#
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: CH4Benchmark
-#
-# comparisons: Specifies the comparisons to perform.
-#
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc: 
       run: True # True to run this comparison
@@ -102,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: False
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Specifies the plots and tables to generate.
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -120,10 +120,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
-#
-# n_cores: Specify the number of cores to use.
-# -1: Use as many cores as possible
-#  N: Use N cores
-#  1: Disable parallelization (use a single core)
-#
+  #
+  # n_cores: Specify the number of cores to use.
+  # -1: Use all available cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
   n_cores: -1
diff --git a/benchmark/config/1yr_fullchem_benchmark.yml b/benchmark/config/1yr_fullchem_benchmark.yml
index 2b62e8f3..bb88c70c 100644
--- a/benchmark/config/1yr_fullchem_benchmark.yml
+++ b/benchmark/config/1yr_fullchem_benchmark.yml
@@ -81,13 +81,13 @@ data:
 # options: Customize the benchmark plot output.
 # 
 options:
-#
-# bmk_type: Specifies the type of benchmark
-#
+  #
+  # bmk_type: Specifies the type of benchmark
+  #
   bmk_type: FullChemBenchmark
-#
-# comparisons: Specifies the comparisons to perform.
-#
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc: 
       run: True # True to run this comparison
@@ -104,9 +104,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: True
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Types of output to generate (plots/tables)
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -123,10 +123,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
-#
-# n_cores: Specify the number of cores to use:
-# -1: Use as many cores as possible
-#  N: Use N cores
-#  1: Disable parallelization (use a single core)
-#
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use all available cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
   n_cores: -1
diff --git a/benchmark/config/1yr_tt_benchmark.yml b/benchmark/config/1yr_tt_benchmark.yml
index 916b01cb..264fed2c 100644
--- a/benchmark/config/1yr_tt_benchmark.yml
+++ b/benchmark/config/1yr_tt_benchmark.yml
@@ -113,8 +113,8 @@ options:
     cons_table: True
   #
   # n_cores: Specify the number of cores to use:
-  # -1: Use as many cores as possible
-  #  N: Use N cores
+  # -1: Use all available cores
+  # -N: Use N cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1

From 958ddd54dea36da87ff3f620068cfe85dfe0c989 Mon Sep 17 00:00:00 2001
From: Bob Yantosca <yantosca@seas.harvard.edu>
Date: Thu, 17 Aug 2023 17:15:59 -0400
Subject: [PATCH 5/6] Now prevent Parallel() commands from executing if
 n_jobs==1

gcpy.benchmark.py
benchmark/modules/run_1yr_fullchem_benchmark.py
benchmark/modules/run_1yr_tt_benchmark.py
- Instead of relying on joblib.Parallel(n_jobs=1) to disable executing
  commands in parallel, we have added if blocks that execute non-parallel
  code when n_jobs=1.  This will help with debugging because error
  messages from upstream packages often get obfuscated by the
  joblib.Parallel() command.

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>
---
 CHANGELOG.md                                  |  1 +
 .../modules/run_1yr_fullchem_benchmark.py     | 88 +++++++++++++------
 benchmark/modules/run_1yr_tt_benchmark.py     | 41 ++++++---
 gcpy/benchmark.py                             | 48 +++++++---
 4 files changed, 128 insertions(+), 50 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0110bb71..c7f084ea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Now use `pypdf` instead of `PyPDF2` in `plot.py` and `util.py`
 - Added coding suggestions made by `pylint` where possible
 - Abstracted and never-nested code from `six_plot` into functions (in `plot.py`)
+- Now add `if` statements to turn of `Parallel()` commands when `n_jobs==1`.
 
 ### Fixed
 - Generalized test for GCHP or GCClassic restart file in `regrid_restart_file.py`
diff --git a/benchmark/modules/run_1yr_fullchem_benchmark.py b/benchmark/modules/run_1yr_fullchem_benchmark.py
index 607e270e..aaa7e4b8 100755
--- a/benchmark/modules/run_1yr_fullchem_benchmark.py
+++ b/benchmark/modules/run_1yr_fullchem_benchmark.py
@@ -687,11 +687,16 @@ def gcc_vs_gcc_mass_table(mon):
                     spcdb_dir=spcdb_dir,
                 )
 
-            # Run in parallel
-            results = Parallel(n_jobs=-1)(
-                delayed(gcc_vs_gcc_mass_table)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gcc_vs_gcc_mass_table)(mon)
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gcc_vs_gcc_mass_table(mon)
 
         # ==================================================================
         # GCC vs GCC operations budgets tables
@@ -729,11 +734,16 @@ def gcc_vs_gcc_ops_budg(mon):
                     dst=gcc_vs_gcc_tablesdir,
                 )
 
-            # Run in parallel
-            results = Parallel(n_jobs=-1)(
-                delayed(gcc_vs_gcc_ops_budg)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gcc_vs_gcc_ops_budg)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gcc_vs_gcc_ops_budg(mon)
 
         # ==================================================================
         # GCC vs GCC aerosols budgets/burdens tables
@@ -1238,10 +1248,16 @@ def gchp_vs_gcc_mass_table(mon):
                     dev_met_extra=devareapath
                 )
 
-            results = Parallel(n_jobs=-1)(
-                delayed(gchp_vs_gcc_mass_table)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gchp_vs_gcc_mass_table)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gchp_vs_gcc_mass_table(mon)
 
         # ==================================================================
         # GCHP vs GCC operations budgets tables
@@ -1288,10 +1304,16 @@ def gchp_vs_gcc_ops_budg(mon):
                     dst=gchp_vs_gcc_tablesdir,
                 )
 
-            results = Parallel(n_jobs=-1)(
-                delayed(gchp_vs_gcc_ops_budg)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gchp_vs_gcc_ops_budg)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gchp_vs_gcc_ops_budg(mon)
 
         # ==================================================================
         # GCHP vs GCC aerosol budgets and burdens tables
@@ -1834,11 +1856,16 @@ def gchp_vs_gchp_mass_table(mon):
                     dev_met_extra=devareapath
                 )
 
-            # Run in parallel
-            results = Parallel(n_jobs=-1)(
-                delayed(gchp_vs_gchp_mass_table)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gchp_vs_gchp_mass_table)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gchp_vs_gchp_mass_table(mon)
 
         # ==================================================================
         # GCHP vs GCHP operations budgets tables
@@ -1887,11 +1914,16 @@ def gchp_vs_gchp_ops_budg(mon):
                     dst=gchp_vs_gchp_tablesdir,
                 )
 
-            # Run in parallel
-            results = Parallel(n_jobs=-1)(
-                delayed(gchp_vs_gchp_ops_budg)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gchp_vs_gchp_ops_budg)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gchp_vs_gchp_ops_budg(mon)
 
         # ==================================================================
         # GCHP vs GCHP aerosol budgets and burdens tables
diff --git a/benchmark/modules/run_1yr_tt_benchmark.py b/benchmark/modules/run_1yr_tt_benchmark.py
index cf371bad..6f736033 100755
--- a/benchmark/modules/run_1yr_tt_benchmark.py
+++ b/benchmark/modules/run_1yr_tt_benchmark.py
@@ -532,11 +532,16 @@ def gcc_vs_gcc_mass_table(mon):
                     spcdb_dir=spcdb_dir,
                 )
 
-            # Run in parallel
-            results = Parallel(n_jobs=-1)(
-                delayed(gcc_vs_gcc_mass_table)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gcc_vs_gcc_mass_table)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gcc_vs_gcc_mass_table(mon)
 
         # ==================================================================
         # GCC vs GCC operations budgets tables
@@ -824,6 +829,17 @@ def gchp_vs_gcc_mass_table(mon):
                         "is_pre_14.0"]
                 )
 
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gchp_vs_gcc_mass_table)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gchp_vs_gcc_mass_table(mon)
+
         # ==================================================================
         # GCHP vs GCC operations budgets tables
         # ==================================================================
@@ -1125,11 +1141,16 @@ def gchp_vs_gchp_mass_table(mon):
                     dev_met_extra=devareapath
                 )
 
-            # Run in parallel
-            results = Parallel(n_jobs=-1)(
-                delayed(gchp_vs_gchp_mass_table)(mon) \
-                for mon in range(bmk_n_months)
-            )
+            # Create tables in parallel
+            # Turn off parallelization if n_jobs==1
+            if n_jobs != 1:
+                results = Parallel(n_jobs=config["options"]["n_cores"])(
+                    delayed(gchp_vs_gchp_mass_table)(mon) \
+                    for mon in range(bmk_n_months)
+                )
+            else:
+                for mon in range(bmk_n_months):
+                    results = gchp_vs_gchp_mass_table(mon)
 
         # ==================================================================
         # GCHP vs GCHP operations budgets tables
diff --git a/gcpy/benchmark.py b/gcpy/benchmark.py
index 8804fbab..aa1cc252 100644
--- a/gcpy/benchmark.py
+++ b/gcpy/benchmark.py
@@ -1498,13 +1498,18 @@ def createplots(filecat):
             )
         return {filecat: cat_diff_dict}
 
+    # --------------------------------------------
     # Create the plots in parallel
-    results = Parallel(n_jobs=n_job)(
-        delayed(createplots)(filecat) for _, filecat in enumerate(catdict)
-    )
-#     # Do not create plots in parallel
-#    for _, filecat in enumerate(catdict):
-#        createplots(filecat)
+    # Turn off parallelization if n_job=1
+    if n_job != 1:
+        results = Parallel(n_jobs=n_job)(
+            delayed(createplots)(filecat)
+            for _, filecat in enumerate(catdict)
+        )
+    else:
+        for _, filecat in enumerate(catdict):
+            results = createplots(filecat)
+    # --------------------------------------------
 
     dict_sfc = {list(result.keys())[0]: result[list(
         result.keys())[0]]['sfc'] for result in results}
@@ -1858,8 +1863,18 @@ def createfile_hco_cat(c):
             diff_dict[c] = diff_emis
             return diff_dict
 
-        results = Parallel(n_jobs=n_job)(delayed(createfile_hco_cat)(c)
-                                         for c in emis_cats)
+        # ---------------------------------------
+        # Create plots in parallel
+        # Turn off parallelization if n_job=1
+        if n_job != 1:
+            results = Parallel(n_jobs=n_job)(
+                delayed(createfile_hco_cat)(c)
+                for c in emis_cats
+            )
+        else:
+            for c in emis_cats:
+                results = createfile_hco_cat(c)
+        # ---------------------------------------
 
         dict_emis = {list(result.keys())[0]: result[list(result.keys())[0]]
                      for result in results}
@@ -1960,10 +1975,19 @@ def createfile_bench_cat(filecat):
             util.add_nested_bookmarks_to_pdf(
                 pdfname, filecat, emisdict, warninglist)
             return catspc
-        results = Parallel(n_jobs=n_job)(
-            delayed(createfile_bench_cat)(filecat)
-            for i, filecat in enumerate(catdict)
-        )
+
+        #------------------------------------------------
+        # Create plots in parallel
+        # Turn of parallalization if n_job=1
+        if n_job != 1:
+            results = Parallel(n_jobs=n_job)(
+                delayed(createfile_bench_cat)(filecat)
+                for _, filecat in enumerate(catdict)
+            )
+        else:
+            for _, filecat in enumerate(catdict):
+                results = createfile_bench_cat(filecat)
+        #------------------------------------------------
 
         allcatspc = [spc for result in results for spc in result]
         # Give warning if emissions species is not assigned a benchmark

From be766a28ed4f18541ce0aabc4921502bfde3c2e1 Mon Sep 17 00:00:00 2001
From: Bob Yantosca <yantosca@seas.harvard.edu>
Date: Wed, 30 Aug 2023 11:50:05 -0400
Subject: [PATCH 6/6] Update comments about n_cores in *_benchmark.yml files

benchmark/cloud/template,1hr_benchmark.yml
benchmark/cloud/template.1mo_benchmark.yml
benchmark/config/1yr_ch4_benchmark.yml
benchmark/config/1yr_fullchem_benchmark.yml
benchmark/config/1yr_tt_benchmark.yml
- Update the comments about n_cores to more accurately reflect
  its functionality.
    -1 uses all $OMP_NUM_THREADS cores,
    -N uses $OMP_NUM_THREADS - (N-1) cores
     1 disables parallelization entirely.

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>
---
 benchmark/cloud/template.1hr_benchmark.yml  | 7 ++++---
 benchmark/cloud/template.1mo_benchmark.yml  | 7 ++++---
 benchmark/config/1yr_ch4_benchmark.yml      | 5 +++--
 benchmark/config/1yr_fullchem_benchmark.yml | 7 ++++---
 benchmark/config/1yr_tt_benchmark.yml       | 7 ++++---
 5 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/benchmark/cloud/template.1hr_benchmark.yml b/benchmark/cloud/template.1hr_benchmark.yml
index 68006695..b3612444 100644
--- a/benchmark/cloud/template.1hr_benchmark.yml
+++ b/benchmark/cloud/template.1hr_benchmark.yml
@@ -121,9 +121,10 @@ options:
       by_spc_cat: True
       by_hco_cat: True
   #
-  # n_cores: Specify the number of cores to use:
-  # -1: Use all available cores
-  # -N: Use N cores
+  # n_cores: Specify the number of cores to use.
+  # -1: Use $OMP_NUM_THREADS         cores
+  # -2: Use $OMP_NUM_THREADS - 1     cores
+  # -N: Use $OMP_NUM_THREADS - (N-1) cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1
diff --git a/benchmark/cloud/template.1mo_benchmark.yml b/benchmark/cloud/template.1mo_benchmark.yml
index 7b9c65e6..792f0d8b 100644
--- a/benchmark/cloud/template.1mo_benchmark.yml
+++ b/benchmark/cloud/template.1mo_benchmark.yml
@@ -121,9 +121,10 @@ options:
       by_spc_cat: True
       by_hco_cat: True
   #
-  # n_cores: Specify the number of cores to use:
-  # -1: Use all available cores
-  # -N: Use N cores
+  # n_cores: Specify the number of cores to use.
+  # -1: Use $OMP_NUM_THREADS         cores
+  # -2: Use $OMP_NUM_THREADS - 1     cores
+  # -N: Use $OMP_NUM_THREADS - (N-1) cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1
diff --git a/benchmark/config/1yr_ch4_benchmark.yml b/benchmark/config/1yr_ch4_benchmark.yml
index 2a656f69..90bc9763 100644
--- a/benchmark/config/1yr_ch4_benchmark.yml
+++ b/benchmark/config/1yr_ch4_benchmark.yml
@@ -122,8 +122,9 @@ options:
       by_hco_cat: True
   #
   # n_cores: Specify the number of cores to use.
-  # -1: Use all available cores
-  # -N: Use N cores
+  # -1: Use $OMP_NUM_THREADS         cores
+  # -2: Use $OMP_NUM_THREADS - 1     cores
+  # -N: Use $OMP_NUM_THREADS - (N-1) cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1
diff --git a/benchmark/config/1yr_fullchem_benchmark.yml b/benchmark/config/1yr_fullchem_benchmark.yml
index bb88c70c..10536861 100644
--- a/benchmark/config/1yr_fullchem_benchmark.yml
+++ b/benchmark/config/1yr_fullchem_benchmark.yml
@@ -124,9 +124,10 @@ options:
       by_spc_cat: True
       by_hco_cat: True
   #
-  # n_cores: Specify the number of cores to use:
-  # -1: Use all available cores
-  # -N: Use N cores
+  # n_cores: Specify the number of cores to use.
+  # -1: Use $OMP_NUM_THREADS         cores
+  # -2: Use $OMP_NUM_THREADS - 1     cores
+  # -N: Use $OMP_NUM_THREADS - (N-1) cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1
diff --git a/benchmark/config/1yr_tt_benchmark.yml b/benchmark/config/1yr_tt_benchmark.yml
index 264fed2c..763f571f 100644
--- a/benchmark/config/1yr_tt_benchmark.yml
+++ b/benchmark/config/1yr_tt_benchmark.yml
@@ -112,9 +112,10 @@ options:
     ste_table: True
     cons_table: True
   #
-  # n_cores: Specify the number of cores to use:
-  # -1: Use all available cores
-  # -N: Use N cores
+  # n_cores: Specify the number of cores to use.
+  # -1: Use $OMP_NUM_THREADS         cores
+  # -2: Use $OMP_NUM_THREADS - 1     cores
+  # -N: Use $OMP_NUM_THREADS - (N-1) cores
   #  1: Disable parallelization (use a single core)
   #
   n_cores: -1