diff --git a/notebooks/compare_ts_and_hist_003.ipynb b/notebooks/compare_ts_and_hist_003.ipynb index 7a5001d..28b24bd 100644 --- a/notebooks/compare_ts_and_hist_003.ipynb +++ b/notebooks/compare_ts_and_hist_003.ipynb @@ -6,8 +6,6 @@ "metadata": {}, "outputs": [], "source": [ - "import yaml\n", - "\n", "import utils" ] }, @@ -15,103 +13,55 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], - "source": [ - "with open(\"diag_metadata.yaml\", mode=\"r\") as fptr:\n", - " diag_metadata_list = yaml.safe_load(fptr)\n", - "\n", - "varnames = utils.get_varnames_from_metadata_list(diag_metadata_list)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Starting year 0001...\n", - "No differences found in year 0001\n", - "Finished 0001\n", + "Checking year 0001...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0001\n", "----\n", - "Starting year 0002...\n", - "No differences found in year 0002\n", - "Finished 0002\n", + "Checking year 0002...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0002\n", "----\n", - "Starting year 0003...\n", - "No differences found in year 0003\n", - "Finished 0003\n", + "Checking year 0003...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0003\n", "----\n", - "Starting year 0004...\n", - "No differences found in year 0004\n", - "Finished 0004\n", + "Checking year 0004...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0004\n", "----\n", - "Starting year 0005...\n", - "Year 0005 time series is not available\n" + "Checking year 0005...\n", + "... checking stream pop.h.nyear1 ...\n", + "Could not find time series for year 0005\n", + "CPU times: user 57.1 s, sys: 2.92 s, total: 1min\n", + "Wall time: 2min 5s\n" ] } ], "source": [ - "casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.003\"\n", - "stream = \"pop.h\"\n", - "\n", - "for year in range(1, 62):\n", - " print(f\"Starting year {year:04}...\")\n", - " ts_found = True\n", - " hist_found = True\n", - " all_same = True\n", - " for diag_metadata in diag_metadata_list:\n", - " varname = diag_metadata[\"varname\"]\n", - " isel_kwargs = diag_metadata.get(\"isel_dict\")\n", - " comp_test = utils.compare_ts_and_hist(\n", - " casename, varname, stream, year, isel_kwargs or {}\n", - " )\n", - " # Error checking (TODO: replace string compare)\n", - "\n", - " # Missing time series data\n", - " if comp_test == \"Can not generate time series dataset\":\n", - " print(f\"Year {year:04} time series is not available\")\n", - " ts_found = False\n", - " break\n", - " if comp_test == \"case_ts does not provide time series files\":\n", - " print(f\"case_ts data for {varname} is not from time series files\")\n", - " ts_found = False\n", - " break\n", - "\n", - " # Missing history file data\n", - " if comp_test == \"History files unavailable\":\n", - " hist_found = False\n", - " break\n", - " if comp_test == \"case_hist does not provide history files\":\n", - " print(f\"case_hist data for {varname} is not from history files\")\n", - " hist_found = False\n", - " break\n", + "%%time\n", "\n", - " # Datasets differ\n", - " if comp_test == \"datasets differ\":\n", - " print(f\"{varname} is different in year {year:04}\")\n", - " all_same = False\n", - "\n", - " # Error checking after running through all variables for a given year\n", - " # (1) If time series data is not available, we are done testing\n", - " if not ts_found:\n", - " break\n", - "\n", - " # (2) If history files are not available, then we have scrubbed those files\n", - " if not hist_found:\n", - " print(\n", - " f\"History files for variables in {year:04} are missing, skipping comparison\"\n", - " )\n", - " print(\"----\")\n", - " continue\n", - "\n", - " # (3) was the data in the time series files identical to that in the history files?\n", - " if all_same:\n", - " print(f\"No differences found in year {year:04}\")\n", - " print(f\"Finished {year:04}\")\n", - " print(\"----\")" + "casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.003\"\n", + "utils.timeseries_and_history_comparison(casename)" ] } ], diff --git a/notebooks/compare_ts_and_hist_004.ipynb b/notebooks/compare_ts_and_hist_004.ipynb index 222d290..2db2b84 100644 --- a/notebooks/compare_ts_and_hist_004.ipynb +++ b/notebooks/compare_ts_and_hist_004.ipynb @@ -6,8 +6,6 @@ "metadata": {}, "outputs": [], "source": [ - "import yaml\n", - "\n", "import utils" ] }, @@ -15,127 +13,103 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], - "source": [ - "with open(\"diag_metadata.yaml\", mode=\"r\") as fptr:\n", - " diag_metadata_list = yaml.safe_load(fptr)\n", - "\n", - "varnames = utils.get_varnames_from_metadata_list(diag_metadata_list)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Starting year 0001...\n", - "No differences found in year 0001\n", - "Finished 0001\n", + "Checking year 0001...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0001\n", "----\n", - "Starting year 0002...\n", - "No differences found in year 0002\n", - "Finished 0002\n", + "Checking year 0002...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0002\n", "----\n", - "Starting year 0003...\n", - "No differences found in year 0003\n", - "Finished 0003\n", + "Checking year 0003...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0003\n", "----\n", - "Starting year 0004...\n", - "No differences found in year 0004\n", - "Finished 0004\n", + "Checking year 0004...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0004\n", "----\n", - "Starting year 0005...\n", - "No differences found in year 0005\n", - "Finished 0005\n", + "Checking year 0005...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0005\n", "----\n", - "Starting year 0006...\n", - "No differences found in year 0006\n", - "Finished 0006\n", + "Checking year 0006...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0006\n", "----\n", - "Starting year 0007...\n", - "No differences found in year 0007\n", - "Finished 0007\n", + "Checking year 0007...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0007\n", "----\n", - "Starting year 0008...\n", - "No differences found in year 0008\n", - "Finished 0008\n", + "Checking year 0008...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0008\n", "----\n", - "Starting year 0009...\n", - "No differences found in year 0009\n", - "Finished 0009\n", + "Checking year 0009...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0009\n", "----\n", - "Starting year 0010...\n", - "No differences found in year 0010\n", - "Finished 0010\n", + "Checking year 0010...\n", + "... checking stream pop.h.nyear1 ...\n", + "... checking stream pop.h.nday1 ...\n", + "... checking stream pop.h ...\n", + "... checking stream cice.h1 ...\n", + "... checking stream cice.h ...\n", + "All variables available in time series for year 0010\n", "----\n", - "Starting year 0011...\n", - "case_ts data for POC_FLUX_100m is not from time series files\n" + "Checking year 0011...\n", + "... checking stream pop.h.nyear1 ...\n", + "Could not find time series for year 0011\n", + "CPU times: user 2min 25s, sys: 11.6 s, total: 2min 36s\n", + "Wall time: 6min 6s\n" ] } ], "source": [ - "casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.004\"\n", - "stream = \"pop.h\"\n", - "\n", - "for year in range(1, 62):\n", - " print(f\"Starting year {year:04}...\")\n", - " ts_found = True\n", - " hist_found = True\n", - " all_same = True\n", - " for diag_metadata in diag_metadata_list:\n", - " varname = diag_metadata[\"varname\"]\n", - " isel_kwargs = diag_metadata.get(\"isel_dict\")\n", - " comp_test = utils.compare_ts_and_hist(\n", - " casename, varname, stream, year, isel_kwargs or {}\n", - " )\n", - " # Error checking (TODO: replace string compare)\n", - "\n", - " # Missing time series data\n", - " if comp_test == \"Can not generate time series dataset\":\n", - " print(f\"Year {year:04} time series is not available\")\n", - " ts_found = False\n", - " break\n", - " if comp_test == \"case_ts does not provide time series files\":\n", - " print(f\"case_ts data for {varname} is not from time series files\")\n", - " ts_found = False\n", - " break\n", - "\n", - " # Missing history file data\n", - " if comp_test == \"History files unavailable\":\n", - " hist_found = False\n", - " break\n", - " if comp_test == \"case_hist does not provide history files\":\n", - " print(f\"case_hist data for {varname} is not from history files\")\n", - " hist_found = False\n", - " break\n", + "%%time\n", "\n", - " # Datasets differ\n", - " if comp_test == \"datasets differ\":\n", - " print(f\"{varname} is different in year {year:04}\")\n", - " all_same = False\n", - "\n", - " # Error checking after running through all variables for a given year\n", - " # (1) If time series data is not available, we are done testing\n", - " if not ts_found:\n", - " break\n", - "\n", - " # (2) If history files are not available, then we have scrubbed those files\n", - " if not hist_found:\n", - " print(\n", - " f\"History files for variables in {year:04} are missing, skipping comparison\"\n", - " )\n", - " print(\"----\")\n", - " continue\n", - "\n", - " # (3) was the data in the time series files identical to that in the history files?\n", - " if all_same:\n", - " print(f\"No differences found in year {year:04}\")\n", - " print(f\"Finished {year:04}\")\n", - " print(\"----\")" + "casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.004\"\n", + "utils.timeseries_and_history_comparison(casename)" ] } ], diff --git a/notebooks/utils/CaseClass.py b/notebooks/utils/CaseClass.py index b090c07..801b9bb 100644 --- a/notebooks/utils/CaseClass.py +++ b/notebooks/utils/CaseClass.py @@ -14,7 +14,9 @@ add_first_date_and_reformat, get_archive_log_dir, get_campaign_popseries_dir, + get_campaign_ciceseries_dir, get_archive_pophist_dir, + get_archive_cicehist_dir, get_rundir, ) from .utils import time_set_mid @@ -66,6 +68,48 @@ def get_co2calc_warning_cnt(self, max_it=4): ############################################################################ + def _get_single_year_timeseries_files(self, year, stream, varname=None): + var_check = True + timeseries_filenames = [] + for filename in self._timeseries_filenames[stream]: + if varname is not None: + var_check = f".{varname}." in filename + if var_check and f".{year:04}" in filename: + timeseries_filenames.extend([filename]) + return timeseries_filenames + + ############################################################################ + + def get_timeseries_files(self, year, stream, varnames=None): + if type(varnames) == str: + varnames = [varnames] + if not (type(varnames) == list or varnames is None): + raise ValueError( + f"varnames = {varnames} which is not None, a string, or a list" + ) + + timeseries_filenames = [] + if varnames: + for varname in varnames: + timeseries_filenames.extend( + self._get_single_year_timeseries_files(year, stream, varname) + ) + else: + timeseries_filenames = self._get_single_year_timeseries_files(year, stream) + + return timeseries_filenames + + ############################################################################ + + def get_history_files(self, year, stream): + return [ + filename + for filename in self._history_filenames[stream] + if f".{year:04}" in filename + ] + + ############################################################################ + def _find_log_files(self): """ Look in rundir and archive for cesm.log, ocn.log, and cpl.log files @@ -91,6 +135,8 @@ def _find_timeseries_files(self): subdirs["pop.h"] = "month_1" subdirs["pop.h.nday1"] = "day_1" subdirs["pop.h.nyear1"] = "year_1" + subdirs["cice.h"] = "month_1" + subdirs["cice.h1"] = "day_1" for stream in ["pop.h", "pop.h.nday1", "pop.h.nyear1"]: files[stream] = [] for casename in self._casenames: @@ -104,6 +150,19 @@ def _find_timeseries_files(self): ) ) files[stream].sort() + for stream in ["cice.h", "cice.h1"]: + files[stream] = [] + for casename in self._casenames: + files[stream].extend( + glob.glob( + os.path.join( + get_campaign_ciceseries_dir(casename), + subdirs[stream], + f"{casename}.{stream}.*.nc", + ) + ) + ) + files[stream].sort() return files ############################################################################ @@ -113,7 +172,7 @@ def _find_hist_files(self): Look in rundir and archive for pop history files """ files = dict() - for stream in ["pop.h", "pop.h.nday1"]: + for stream in ["pop.h", "pop.h.nday1", "pop.h.nyear1"]: files[stream] = [] for rootdir in [get_archive_pophist_dir, get_rundir]: for casename in self._casenames: @@ -125,6 +184,18 @@ def _find_hist_files(self): ) ) files[stream].sort() + for stream in ["cice.h", "cice.h1"]: + files[stream] = [] + for rootdir in [get_archive_cicehist_dir, get_rundir]: + for casename in self._casenames: + files[stream].extend( + glob.glob( + os.path.join( + rootdir(casename), f"{casename}.{stream}.0*.nc" + ) + ) + ) + files[stream].sort() return files ############################################################################ @@ -255,7 +326,7 @@ def gen_dataset( if type(varnames) == str: varnames = [varnames] if type(varnames) != list: - raise ValueError(f"{casenames} is not a string or list") + raise ValueError(f"{varnames} is not a string or list") if stream not in self._dataset_files: self._dataset_files[stream] = dict() @@ -292,20 +363,21 @@ def gen_dataset( if year not in self._dataset_files[stream]: self._dataset_files[stream][year] = dict() self._dataset_src[stream][year] = dict() - self._dataset_files[stream][year][varname] = [ - filename - for filename in self._timeseries_filenames[stream] - if f".{varname}." in filename and f".{year:04}" in filename - ] - self._dataset_src[stream][year][varname] = "time series" - timeseries_filenames.extend(self._dataset_files[stream][year][varname]) - + self._dataset_files[stream][year][varname] = self.get_timeseries_files( + year, stream, varnames + ) + if self._dataset_files[stream][year][varname]: + self._dataset_src[stream][year][varname] = "time series" + timeseries_filenames.extend( + self._dataset_files[stream][year][varname] + ) if timeseries_filenames: ds_timeseries_per_var.append( xr.open_mfdataset(timeseries_filenames, **open_mfdataset_kwargs,)[ [varname] + _vars_to_keep ] ) + if ds_timeseries_per_var: ds_timeseries = xr.merge(ds_timeseries_per_var) tb_name_ts = ds_timeseries["time"].attrs["bounds"] @@ -330,13 +402,12 @@ def gen_dataset( if year not in self._dataset_files[stream]: self._dataset_files[stream][year] = dict() self._dataset_src[stream][year] = dict() - self._dataset_files[stream][year][varname] = [ - filename - for filename in self._history_filenames[stream] - if f".{year:04}" in filename - ] - history_filenames.extend(self._dataset_files[stream][year][varname]) - self._dataset_src[stream][year][varname] = "hist" + self._dataset_files[stream][year][varname] = self.get_history_files( + year, stream + ) + if self._dataset_files[stream][year][varname]: + self._dataset_src[stream][year][varname] = "hist" + history_filenames.extend(self._dataset_files[stream][year][varname]) if history_filenames: ds_history = xr.open_mfdataset(history_filenames, **open_mfdataset_kwargs,)[ diff --git a/notebooks/utils/__init__.py b/notebooks/utils/__init__.py index c92d39c..1edab76 100644 --- a/notebooks/utils/__init__.py +++ b/notebooks/utils/__init__.py @@ -8,4 +8,5 @@ from .Plotting import summary_plot_maps from .Plotting import trend_plot from .utils import get_varnames_from_metadata_list +from .utils import timeseries_and_history_comparison from .compare_ts_and_hist import compare_ts_and_hist diff --git a/notebooks/utils/compare_ts_and_hist.py b/notebooks/utils/compare_ts_and_hist.py index 958b2f8..60edba6 100755 --- a/notebooks/utils/compare_ts_and_hist.py +++ b/notebooks/utils/compare_ts_and_hist.py @@ -3,61 +3,60 @@ A script to verify that converting from history files to time series worked as expected """ +import xarray as xr + from .CaseClass import CaseClass -def compare_ts_and_hist(casename, varname, stream, year, isel_kwargs): +def compare_ts_and_hist( + casename, stream, year, exclude_vars=["time_bound", "time_bounds"] +): """ - Generate two CaseClass objects from the same casename, one - from time series output and the other from history files. - - Compare the datasets generated by these two objects; they - should be identical. Possible error states: - - 1. can not find time series {stream} from {year} for {varname} - 2. can not find history files {stream} from {year} for {varname} - 3. datasets differ + Generate a CaseClass object from a given casename. For a given stream + and year, open the history files from the case. Then loop through the + variables (excluding time_bound in POP and time_bounds in CICE) and + verify that those fields are available in time series. """ - case_ts = CaseClass(casename) - case_hist = CaseClass(casename) - case_hist._timeseries_filenames[stream] = [] + # Set some defaults to pass to open_mfdataset, then apply kwargs argument + open_mfdataset_kwargs = dict() + # data_vars="minimal", to avoid introducing time dimension to time-invariant fields + open_mfdataset_kwargs["data_vars"] = "minimal" + # compat="override", to skip var consistency checks (for speed) + open_mfdataset_kwargs["compat"] = "override" + # coords="minimal", because coords cannot be default="different" if compat="override" + open_mfdataset_kwargs["coords"] = "minimal" + # parallel=True to open files in parallel + open_mfdataset_kwargs["parallel"] = True - # (1) generate datasets - # (a) time series - try: - ds_ts = case_ts.gen_dataset( - varname, stream, start_year=year, end_year=year, quiet=True - ) - except ValueError: - return "Can not generate time series dataset" + found_all = True - # (b) history files - try: - ds_hist = case_hist.gen_dataset( - varname, stream, start_year=year, end_year=year, quiet=True - ) - except ValueError: - return "History files unavailable" + case = CaseClass(casename) + # Return if no time series is available + if len(case.get_timeseries_files(year, stream)) == 0: + return "no time series" - # (2) Check sources of data being read - # (a) case_ts should return data from time series - data_source = case_ts.get_dataset_source(stream, year, varname) - if data_source != "time series": - return "case_ts does not provide time series files" + # Return if no history files are available + history_filenames = case.get_history_files(year, stream) + if len(history_filenames) == 0: + return "no history" - # (b) case_ts should return data from history files - # note: this condition should never be met, as it will be - # caught in (1b) - data_source = case_hist.get_dataset_source(stream, year, varname) - if data_source != "hist": - return "case_hist does not provide history files" + # Open history files to build dataset + ds_hist = xr.open_mfdataset(history_filenames, **open_mfdataset_kwargs) + vars_to_check = [ + var + for var in ds_hist.data_vars + if "time" in ds_hist[var].coords and not var in exclude_vars + ] - # (3) Compare datasets - da_hist = ds_hist[varname].isel(**isel_kwargs) - da_ts = ds_ts[varname].isel(**isel_kwargs) - if not da_hist.identical(da_ts): - return "datasets differ" + # Look for each variable in time series + for var in vars_to_check: + if len(case.get_timeseries_files(year, stream, var)) == 0: + print(f"No time series files for {var} in year {year:04}") + found_all = False + # Return "same" if all variables were found, otherwise return "datasets differ" + if not found_all: + return "datasets differ" return "same" diff --git a/notebooks/utils/config.py b/notebooks/utils/config.py index 88996d2..d56fbb3 100644 --- a/notebooks/utils/config.py +++ b/notebooks/utils/config.py @@ -37,6 +37,29 @@ def get_campaign_popseries_dir(casename): ################################################################################ +def get_campaign_ciceseries_dir(casename): + freq_name = dict() + return os.path.join( + os.sep, + "glade", + "campaign", + "cesm", + "development", + "bgcwg", + "projects", + "hi-res_JRA", + "cases", + casename, + "output", + "ice", + "proc", + "tseries", + ) + + +################################################################################ + + def get_archive_pophist_dir(casename): return os.path.join( os.sep, "glade", "scratch", user, "archive", casename, "ocn", "hist" @@ -46,6 +69,15 @@ def get_archive_pophist_dir(casename): ################################################################################ +def get_archive_cicehist_dir(casename): + return os.path.join( + os.sep, "glade", "scratch", user, "archive", casename, "ice", "hist" + ) + + +################################################################################ + + def get_archive_log_dir(casename): return os.path.join(os.sep, "glade", "scratch", user, "archive", casename, "logs") diff --git a/notebooks/utils/utils.py b/notebooks/utils/utils.py index dddfbaf..a877436 100644 --- a/notebooks/utils/utils.py +++ b/notebooks/utils/utils.py @@ -6,6 +6,9 @@ import numpy as np import xarray as xr +from . import CaseClass +from .compare_ts_and_hist import compare_ts_and_hist + def repl_coord(coordname, ds1, ds2): """ @@ -100,3 +103,43 @@ def get_varnames_from_metadata_list(diag_metadata_list): if diag_metadata["varname"] not in varnames: varnames.append(diag_metadata["varname"]) return varnames + + +def timeseries_and_history_comparison(casename): + case = CaseClass(casename) + for year in range(1, 62): + has_ts = True + found_all = True + print(f"Checking year {year:04}...") + for stream in ["pop.h.nyear1", "pop.h.nday1", "pop.h", "cice.h1", "cice.h"]: + has_hist = True + # There is no cice.h1 time series for 0001 so skip check + if stream == "cice.h1" and year == 1: + continue + # Run test + print(f"... checking stream {stream} ...") + comp_test = compare_ts_and_hist(casename, stream, year) + # Check ends when there are no history files for comparison + if comp_test == "no time series": + has_ts = False + break + + # Skip years when there are no history files + # (Assume those years were already checked prior to deleting history files) + if comp_test == "no history": + print( + f"Skipping stream {stream} for year {year:04} because there are no history files" + ) + has_hist = False + continue + + found_all = found_all and (comp_test == "same") + + if not has_ts: + print(f"Could not find time series for year {year:04}") + break + if has_hist and found_all: + print(f"All variables available in time series for year {year:04}") + else: + print(f"Could not find time series for all variables in year {year:04}") + print("----")