Skip to content

Commit

Permalink
Merge templates (#118)
Browse files Browse the repository at this point in the history 
* merge of csv2bufr-templates module into this module.

* Flake 8, new line at end of tests.

* Logging and exception handling added to csv2bufr.templates.

* Changes following review - better/consistent use of pathlib.

* Doc strings + annotations added.
  • Loading branch information
@david-i-berry
david-i-berry authored Sep 5, 2023
1 parent 9f3b18e commit 4a5cdae
Show file tree
Hide file tree
Showing 9 changed files with 337 additions and 4 deletions.
3 changes: 0 additions & 3 deletions .idea/.gitignore
View file

This file was deleted.

2 changes: 1 addition & 1 deletion csv2bufr/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import click

from csv2bufr import __version__, BUFRMessage, transform as transform_csv
import csv2bufr_templates as c2bt
import csv2bufr.templates as c2bt

THISDIR = os.path.dirname(os.path.realpath(__file__))
MAPPINGS = f"{THISDIR}{os.sep}resources{os.sep}mappings"
Expand Down
86 changes: 86 additions & 0 deletions csv2bufr/templates/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
###############################################################################
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
###############################################################################
import json
import logging
import os
from pathlib import Path
from typing import Union

TEMPLATE_DIRS = []

LOGGER = logging.getLogger(__name__)

# Set user defined location first
if 'CSV2BUFR_TEMPLATES' in os.environ:
TEMPLATE_DIRS.append(Path(os.environ['CSV2BUFR_TEMPLATES']))

# Now add defaults
TEMPLATE_DIRS.append(Path(__file__).resolve().parent / 'resources')


def load_template(template_name: str) -> Union[dict, None]:
"""
Checks whether specified template exists and loads file.
Returns none and prints a warning if no template found.
:param template_name: The name of the template (without file extension)
to load.
:returns: BUFR template as a dictionary or none in the case of template
not found.
"""
template = None
# iterate over directories and load file
for dir_ in TEMPLATE_DIRS:
try:
template_file = dir_ / f"{template_name}.json"
if template_file.is_file():
with template_file.open() as fh:
template = json.load(fh)
break
except Exception as e:
LOGGER.warning(f"Error raised loading csv2bufr templates: {e}.")

if template is None:
LOGGER.warning(f"Requested template '{template_name}' not found." +
f" Search path = {TEMPLATE_DIRS}. Please update " +
"search path (e.g. 'export CSV2BUFR_TEMPLATE=...')"
)

return template


def list_templates() -> list:
"""
:returns: List of known templates in search path (CSV2BUFR_TEMPLATES).
An empty list is return if no templates are found.
"""
templates = []
for dir_ in TEMPLATE_DIRS:
try:
for template in dir_.iterdir():
if template.suffix == ".json":
templates.append(template.stem)
except Exception as e:
LOGGER.warning(f"Error raised listing csv2bufr templates: {e}." +
"Directory skipped.")

return templates
82 changes: 82 additions & 0 deletions csv2bufr/templates/resources/aws-template.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"inputShortDelayedDescriptorReplicationFactor": [],
"inputDelayedDescriptorReplicationFactor": [1,1],
"inputExtendedDelayedDescriptorReplicationFactor": [],
"number_header_rows": 1,
"column_names_row": 1,
"quoting": "QUOTE_NONE",
"header":[
{"eccodes_key": "edition", "value": "const:4"},
{"eccodes_key": "masterTableNumber", "value": "const:0"},
{"eccodes_key": "bufrHeaderCentre", "value": "const:0"},
{"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"},
{"eccodes_key": "updateSequenceNumber", "value": "const:0"},
{"eccodes_key": "dataCategory", "value": "const:0"},
{"eccodes_key": "internationalDataSubCategory", "value": "const:2"},
{"eccodes_key": "masterTablesVersionNumber", "value": "const:30"},
{"eccodes_key": "numberOfSubsets", "value": "const:1"},
{"eccodes_key": "observedData", "value": "const:1"},
{"eccodes_key": "compressedData", "value": "const:0"},
{"eccodes_key": "typicalYear", "value": "data:year"},
{"eccodes_key": "typicalMonth", "value": "data:month"},
{"eccodes_key": "typicalDay", "value": "data:day"},
{"eccodes_key": "typicalHour", "value": "data:hour"},
{"eccodes_key": "typicalMinute", "value": "data:minute"},
{"eccodes_key": "unexpandedDescriptors", "value":"array:301150, 307096"}
],
"data": [
{"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:wsi_series"},
{"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:wsi_issuer"},
{"eccodes_key": "#1#wigosIssueNumber", "value":"data:wsi_issue_number"},
{"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:wsi_local"},
{"eccodes_key": "#1#latitude", "value": "data:latitude"},
{"eccodes_key": "#1#longitude", "value": "data:longitude"},
{"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:station_height_above_msl"},
{"eccodes_key": "#1#heightOfBarometerAboveMeanSeaLevel", "value":"data:barometer_height_above_msl"},
{"eccodes_key": "#1#blockNumber", "value": "data:wmo_block_number"},
{"eccodes_key": "#1#stationNumber", "value": "data:wmo_station_number"},
{"eccodes_key": "#1#stationType", "value": "data:station_type"},
{"eccodes_key": "#1#year", "value": "data:year"},
{"eccodes_key": "#1#month", "value": "data:month"},
{"eccodes_key": "#1#day", "value": "data:day"},
{"eccodes_key": "#1#hour", "value": "const:0", "valid_min": "const:10"},
{"eccodes_key": "#1#minute", "value": "data:minute"},
{"eccodes_key": "#1#nonCoordinatePressure", "value": "data:station_pressure"},
{"eccodes_key": "#1#pressureReducedToMeanSeaLevel", "value": "data:msl_pressure"},
{"eccodes_key": "#1#nonCoordinateGeopotentialHeight", "value": "data:geopotential_height"},
{"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:thermometer_height"},
{"eccodes_key": "#1#airTemperature", "value": "data:air_temperature"},
{"eccodes_key": "#1#dewpointTemperature", "value": "data:dewpoint_temperature"},
{"eccodes_key": "#1#relativeHumidity", "value": "data:relative_humidity"},
{"eccodes_key": "#1#methodOfStateOfGroundMeasurement", "value": "data:method_of_ground_state_measurement"},
{"eccodes_key": "#1#stateOfGround", "value": "data:ground_state"},
{"eccodes_key": "#1#methodOfSnowDepthMeasurement", "value": "data:method_of_snow_depth_measurement"},
{"eccodes_key": "#1#totalSnowDepth", "value": "data:snow_depth"},
{"eccodes_key": "#1#precipitationIntensityHighAccuracy", "value": "data:precipitation_intensity"},
{"eccodes_key": "#8#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:anemometer_height"},
{"eccodes_key": "#3#timeSignificance", "value": "const:2"},
{"eccodes_key": "#6#timePeriod", "value": "data:time_period_of_wind"},
{"eccodes_key": "#1#windDirection", "value": "data:wind_direction"},
{"eccodes_key": "#1#windSpeed", "value": "data:wind_speed"},
{"eccodes_key": "#7#timePeriod", "value": "const:-10"},
{"eccodes_key": "#1#maximumWindGustDirection", "value": "data:maximum_wind_gust_direction_10_minutes"},
{"eccodes_key": "#1#maximumWindGustSpeed", "value": "data:maximum_wind_gust_speed_10_minutes"},
{"eccodes_key": "#8#timePeriod", "value": "const:-60"},
{"eccodes_key": "#2#maximumWindGustDirection", "value": "data:maximum_wind_gust_direction_1_hour"},
{"eccodes_key": "#2#maximumWindGustSpeed", "value": "data:maximum_wind_gust_speed_1_hour"},
{"eccodes_key": "#9#timePeriod", "value": "const:-180"},
{"eccodes_key": "#3#maximumWindGustDirection", "value": "data:maximum_wind_gust_direction_3_hours"},
{"eccodes_key": "#3#maximumWindGustSpeed", "value": "data:maximum_wind_gust_speed_3_hours"},
{"eccodes_key": "#8#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:rain_sensor_height"},
{"eccodes_key": "#17#timePeriod", "value": "const:-1"},
{"eccodes_key": "#1#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_1_hour"},
{"eccodes_key": "#18#timePeriod", "value": "const:-3"},
{"eccodes_key": "#2#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_3_hours"},
{"eccodes_key": "#19#timePeriod", "value": "const:-6"},
{"eccodes_key": "#3#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_6_hours"},
{"eccodes_key": "#20#timePeriod", "value": "const:-12"},
{"eccodes_key": "#4#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_12_hours"},
{"eccodes_key": "#21#timePeriod", "value": "const:-24"},
{"eccodes_key": "#5#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_24_hours"}
]
}
84 changes: 84 additions & 0 deletions csv2bufr/templates/resources/daycli-template.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
{
"inputShortDelayedDescriptorReplicationFactor": [],
"inputDelayedDescriptorReplicationFactor": [],
"inputExtendedDelayedDescriptorReplicationFactor": [],
"number_header_rows": 1,
"column_names_row": 1,
"wigos_station_identifier": "data:wigos_station_identifier",
"header": [
{"eccodes_key": "edition", "value": "const:4"},
{"eccodes_key": "masterTableNumber", "value": "const:0"},
{"eccodes_key": "bufrHeaderCentre", "value": "const:0"},
{"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"},
{"eccodes_key": "updateSequenceNumber", "value": "const:0"},
{"eccodes_key": "dataCategory", "value": "const:0"},
{"eccodes_key": "internationalDataSubCategory", "value": "const:6"},
{"eccodes_key": "masterTablesVersionNumber", "value": "const:38"},
{"eccodes_key": "typicalYear", "value": "data:year"},
{"eccodes_key": "typicalMonth", "value": "data:month"},
{"eccodes_key": "typicalDay", "value": "data:day"},
{"eccodes_key": "typicalHour", "value": "const:23"},
{"eccodes_key": "typicalMinute", "value": "const:59"},
{"eccodes_key": "numberOfSubsets", "value": "const:1"},
{"eccodes_key": "observedData", "value": "const:1"},
{"eccodes_key": "compressedData", "value": "const:0"},
{"eccodes_key": "unexpandedDescriptors", "value": "array:307075"}
],
"data": [
{"eccodes_key": "#1#wigosIdentifierSeries", "value": "data:wsi_series"},
{"eccodes_key": "#1#wigosIssuerOfIdentifier", "value": "data:wsi_issuer"},
{"eccodes_key": "#1#wigosIssueNumber", "value": "data:wsi_issue"},
{"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value": "data:wsi_local"},
{"eccodes_key": "#1#latitude", "value": "data:lat", "valid_min": "const:-90.0", "valid_max": "const:90.0"},
{"eccodes_key": "#1#longitude", "value": "data:lon", "valid_min": "const:-180.0", "valid_max": "const:180.0"},
{"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value": "data:elev"},
{"eccodes_key": "#1#methodUsedToCalculateTheAverageDailyTemperature", "value": "const:0"},
{"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:1800", "valid_max": "const:2100"},
{"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:1", "valid_max": "const:12"},
{"eccodes_key": "#1#day", "value": "data:day", "valid_min": "const:1", "valid_max": "const:31"},
{"eccodes_key": "#1#timePeriod", "value": "const:0"},
{"eccodes_key": "#1#hour", "value": "const:0"},
{"eccodes_key": "#1#minute", "value": "const:0"},
{"eccodes_key": "#1#minute", "value": "const:0"},
{"eccodes_key": "#1#second", "value": "const:0"},
{"eccodes_key": "#1#totalAccumulatedPrecipitation", "value": "data:total_precipitation_or_total_water_equivalent"},
{"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField", "value": "const:7"},
{"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#2#timePeriod", "value": "const:0"},
{"eccodes_key": "#2#hour", "value": "const:0"},
{"eccodes_key": "#2#minute", "value": "const:0"},
{"eccodes_key": "#2#second", "value": "const:0"},
{"eccodes_key": "#1#depthOfFreshSnow->associatedField", "value": "const:5"},
{"eccodes_key": "#1#depthOfFreshSnow->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#3#timePeriod", "value": "const:0"},
{"eccodes_key": "#3#hour", "value": "const:0"},
{"eccodes_key": "#3#minute", "value": "const:0"},
{"eccodes_key": "#3#second", "value": "const:0"},
{"eccodes_key": "#1#totalSnowDepth->associatedField", "value": "const:5"},
{"eccodes_key": "#1#totalSnowDepth->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#4#timePeriod", "value": "const:0"},
{"eccodes_key": "#4#hour", "value": "const:0"},
{"eccodes_key": "#4#minute", "value": "const:0"},
{"eccodes_key": "#4#second", "value": "const:0"},
{"eccodes_key": "#1#firstOrderStatistics", "value": "const:2"},
{"eccodes_key": "#1#airTemperature", "value": "data:maximum_temperature_at_height_and_over_period_specified", "scale": "const:0", "offset": "const:273.15"},
{"eccodes_key": "#1#airTemperature->associatedField", "value": "const:7"},
{"eccodes_key": "#1#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#5#timePeriod", "value": "const:0"},
{"eccodes_key": "#5#hour", "value": "const:0"},
{"eccodes_key": "#5#minute", "value": "const:0"},
{"eccodes_key": "#5#second", "value": "const:0"},
{"eccodes_key": "#2#firstOrderStatistics", "value": "const:3"},
{"eccodes_key": "#2#airTemperature", "value": "data:minimum_temperature_at_height_and_over_period_specified", "scale": "const:0", "offset": "const:273.15"},
{"eccodes_key": "#2#airTemperature->associatedField", "value": "const:7"},
{"eccodes_key": "#2#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"},
{"eccodes_key": "#6#timePeriod", "value": "const:0"},
{"eccodes_key": "#6#hour", "value": "const:0"},
{"eccodes_key": "#6#minute", "value": "const:0"},
{"eccodes_key": "#6#second", "value": "const:0"},
{"eccodes_key": "#3#firstOrderStatistics", "value": "const:4"},
{"eccodes_key": "#3#airTemperature", "value": "data:mean_air_temperature", "scale": "const:0", "offset": "const:273.15"},
{"eccodes_key": "#3#airTemperature->associatedField", "value": "const:7"},
{"eccodes_key": "#3#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}
],
}
0 csv2bufr/resources/mappings/daycli.json → csv2bufr/templates/resources/daycli.json
View file
File renamed without changes.
Loading

0 comments on commit 4a5cdae

Please sign in to comment.