-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #24 from ecmwf-projects/cuon-speed-up
Cuon speed up
- Loading branch information
Showing
5 changed files
with
122 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
from pathlib import Path | ||
|
||
import cftime | ||
import h5netcdf | ||
import numpy | ||
import pandas | ||
from matplotlib import pyplot | ||
|
||
from cdsobs.constants import TIME_UNITS | ||
from cdsobs.ingestion.readers.cuon import _maybe_concat_chars, get_cuon_stations | ||
from cdsobs.utils.logutils import get_logger | ||
|
||
logger = get_logger(__name__) | ||
|
||
|
||
def plot_station_number(): | ||
stations = get_cuon_stations() | ||
stations["start of records"] = cftime.num2date( | ||
stations["start of records"], units=TIME_UNITS | ||
) | ||
stations["end of records"] = cftime.num2date( | ||
stations["end of records"], units=TIME_UNITS | ||
) | ||
print(stations.head().to_string()) | ||
months = pandas.date_range("1905-01-01", "2023-12-31", freq="MS") | ||
station_number = pandas.Series(index=months) | ||
for monthdate in months: | ||
station_record_has_started = stations["start of records"] <= monthdate | ||
station_record_has_not_ended = stations["end of records"] >= monthdate | ||
station_number.loc[monthdate] = numpy.logical_and( | ||
station_record_has_started, station_record_has_not_ended | ||
).sum() | ||
|
||
station_number.plot() | ||
pyplot.show() | ||
|
||
|
||
def get_char_var_data(inc_group, variable): | ||
return _maybe_concat_chars(inc_group[variable][:]) | ||
|
||
|
||
def check_primary_keys_consistency(): | ||
idir = Path("/data/public/converted_v19") | ||
for ipath in idir.glob("*.nc"): | ||
logger.info(f"Checking {ipath}") | ||
try: | ||
check_primary_keys_consistency_file(ipath) | ||
except Exception as e: | ||
logger.info(f"Exception captured for {ipath}: {e}") | ||
|
||
|
||
def check_primary_keys_consistency_file(ipath): | ||
with h5netcdf.File(ipath) as inc: | ||
station_table = inc.groups["station_configuration"] | ||
header_table = inc.groups["header_table"] | ||
observations_table = inc.groups["observations_table"] | ||
station_ids_station_table = get_char_var_data(station_table, "primary_id") | ||
station_ids_header_table = get_char_var_data(header_table, "primary_station_id") | ||
ids_ok = set(station_ids_station_table) == set(station_ids_header_table) | ||
if not ids_ok: | ||
logger.warning(f"Station ids wrong for {ipath}") | ||
records_ok = set(station_table["record_number"][:]) == set( | ||
header_table["station_record_number"][:] | ||
) | ||
if not records_ok: | ||
logger.warning(f"Station record number wrong for {ipath}") | ||
report_ids_observations_table = get_char_var_data( | ||
observations_table, "report_id" | ||
) | ||
report_ids_header_table = get_char_var_data(header_table, "report_id") | ||
report_ids_ok = set(report_ids_observations_table) == set( | ||
report_ids_header_table | ||
) | ||
if not report_ids_ok: | ||
logger.warning(f"Station record ids wrong for {ipath}") | ||
|
||
|
||
if __name__ == "__main__": | ||
check_primary_keys_consistency() |