Now allow specification of the number of cores for benchmark plotting jobs

CHANGELOG.md

@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Added new routine `verify_variable_type` function in `gcpy/util.py`
 - Added new routine `format_number_for_table` in `util.py`
 - Added BrSALA and BrSALC to `emission_species.yml`
+- Added `options:n_cores` to all benchmark YAML config files
 - Added `ENCODING = "UTF-8"` to `gcpy/constants.py`
 
 ### Changed
@@ -39,10 +40,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Routine `print_totals` now prints small and/or large numbers in scientific notation
 - Truncate names in benchmark & emissions tables to improve readability
 - Add TransportTracers species names to `gcpy/emissions_*.yml` files
+- Now pass `n_job=config["options"]["n_cores"]` to benchmark plotting routines
 - Updated `docs/environment_files/environment.yml` for MambaForge (also added `gridspec`)
 - Now use `pypdf` instead of `PyPDF2` in `plot.py` and `util.py`
 - Added coding suggestions made by `pylint` where possible
 - Abstracted and never-nested code from `six_plot` into functions (in `plot.py`)
+- Now add `if` statements to turn of `Parallel()` commands when `n_jobs==1`.
 
 ### Fixed
 - Generalized test for GCHP or GCClassic restart file in `regrid_restart_file.py`

benchmark/cloud/template.1hr_benchmark.yml

@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: False
       resolution: c24
 #
-# options: Specify the types of comparisons to perform
+# options: Customizes the benchmark plot output.
 #
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: FullChemBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc:
       run: False
@@ -92,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: False
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: False
     plot_emis: False
@@ -110,3 +120,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use all available cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1

benchmark/cloud/template.1mo_benchmark.yml

@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: False
       resolution: c24
 #
-# options: Specify the types of comparisons to perform
+# options: Customizes the benchmark plot output.
 #
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: FullChemBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc:
       run: False
@@ -92,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: False
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -110,3 +120,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use all available cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1

benchmark/config/1mo_benchmark.yml

@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,13 +76,19 @@ data:
       is_pre_14.0: False
       resolution: c24
 #
-# options: Specify the types of comparisons to perform
-#   
+# options: Customizes the benchmark plot output.
+#
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: FullChemBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc: 
-      run: True # True to run this comparison
+      run: True
       dir: GCC_version_comparison
       tables_subdir: Tables
     gchp_vs_gcc: 
@@ -92,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs: 
       run: True
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-# 
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -110,3 +120,10 @@ options:
     plot_options: # Plot concentrations and emissions by category?
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use all avaiable cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1

benchmark/config/1yr_ch4_benchmark.yml

@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
-# (4) If necessary, edit labels for the dev and ref versions
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and tables,
+#     as well as other plotting options.
+# (4) If necessary, edit labels for the dev and ref versions.
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: False              # for gcpy_test_data, edit if needed
       resolution: c24                 # for gcpy_test_data, edit if needed   
 #
-# options: Specify the types of comparisons to perform
+# options: Customize the benchmark plot output.
 # 
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: CH4Benchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc: 
       run: True # True to run this comparison
@@ -92,9 +102,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: False
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -110,3 +120,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use.
+  # -1: Use all available cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1

benchmark/config/1yr_fullchem_benchmark.yml

@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
+#
+# (1) Edit the path variables so that they point to folders
+#     containing model data
 # (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables as well as other plotting options.
 # (4) If necessary, edit labels for the dev and ref versions
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -74,10 +78,16 @@ data:
       is_pre_14.0: False              # for gcpy_test_data, edit if needed
       resolution: c24                 # for gcpy_test_data, edit if needed   
 #
-# options: Specify the types of comparisons to perform
+# options: Customize the benchmark plot output.
 # 
 options:
+  #
+  # bmk_type: Specifies the type of benchmark
+  #
   bmk_type: FullChemBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc: 
       run: True # True to run this comparison
@@ -94,9 +104,9 @@ options:
     gchp_vs_gcc_diff_of_diffs:
       run: True
       dir: GCHP_GCC_diff_of_diffs
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Types of output to generate (plots/tables)
+  #
   outputs:
     plot_conc: True
     plot_emis: True
@@ -113,3 +123,10 @@ options:
     plot_options:
       by_spc_cat: True
       by_hco_cat: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use all available cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1

benchmark/config/1yr_tt_benchmark.yml

@@ -2,10 +2,14 @@
 # =====================================================================
 # Benchmark configuration file (**EDIT AS NEEDED**)
 # customize in the following manner:
-# (1) Edit the path variables so that they point to folders w/ model data
-# (2) Edit the version strings for each benchmark simulation
-# (3) Edit the switches that turn on/off creating of plots and tables
-# (4) If necessary, edit labels for the dev and ref versions
+#
+# (1) Edit the path variables so that they point to folders w/
+#     containing model data.
+# (2) Edit the version strings for each benchmark simulation.
+# (3) Edit the switches that turn on/off creating of plots and
+#     tables, as well as other plotting options.
+# (4) If necessary, edit labels for the dev and ref versions.
+#
 # Note: When doing GCHP vs GCC comparisions gchp_dev will be compared
 # to gcc_dev (not gcc_ref!). This ensures consistency in version names
 # when doing GCHP vs GCC diff-of-diffs (mps, 6/27/19)
@@ -72,10 +76,16 @@ data:
       is_pre_14.0: True               # for gcpy_test_data, edit if needed
       resolution: c48                 # for gcpy_test_data, edit if needed
 #
-# options: Specify the types of comparisons to perform
+# options: Customize the benchmark plot output.
 #
 options:
+  #
+  # bmk_type: Specifies the type of benchmark.
+  #
   bmk_type: TransportTracersBenchmark
+  #
+  # comparisons: Specifies the comparisons to perform.
+  #
   comparisons:
     gcc_vs_gcc:
       run: True
@@ -90,9 +100,9 @@ options:
       dir: GCHP_version_comparison
       tables_subdir: Tables
     # GCHP vs GCC diff of diffs not included in 1-yr tt benchmark
-#
-# outputs: Types of output to generate (plots/tables)
-#
+  #
+  # outputs: Specifies the plots and tables to generate.
+  #
   outputs:
     plot_conc: True
     plot_wetdep: True
@@ -101,3 +111,10 @@ options:
     mass_table: True
     ste_table: True
     cons_table: True
+  #
+  # n_cores: Specify the number of cores to use:
+  # -1: Use all available cores
+  # -N: Use N cores
+  #  1: Disable parallelization (use a single core)
+  #
+  n_cores: -1