From bf8cef2aefb738c2de8c4353f0ffcb0b6d6f70b7 Mon Sep 17 00:00:00 2001 From: garciam Date: Thu, 18 Jul 2024 12:50:13 +0200 Subject: [PATCH 01/20] add wetness and Air temperature negative quasisystematic uncertainty --- .../service_definition.yml | 11 ++++++++++- tests/conftest.py | 2 +- tests/test_get_forms_jsons.py | 4 +++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml index 41a47e3..757bb3f 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml @@ -60,6 +60,7 @@ out_columns_order: - soil_temperature_processing_level_quality_flag - wind_speed_2_meters_from_earth_surface - wind_speed_2_meters_from_earth_surface_quality_flag +- wetness - wetness_quality_flag - accumulated_precipitation - daily_global_solar_radiation_rates @@ -787,7 +788,7 @@ sources: latitude: latitude|station_configuration longitude: longitude|station_configuration p_monthly_calc: accumulated_precipitation - quasisys_n: air_temperature_negative_quasisystematic_uncertainty + quasisys_m: air_temperature_negative_quasisystematic_uncertainty quasisys_p: air_temperature_positive_quasisystematic_uncertainty random_m: air_temperature_negative_random_uncertainty random_p: air_temperature_positive_random_uncertainty @@ -1027,6 +1028,7 @@ sources: terr_m: air_temperature_negative_total_uncertainty terr_p: air_temperature_positive_total_uncertainty wbanno: primary_station_id + wetness: wetness wet_flag: wetness_quality_flag wind_1_5: wind_speed_2_meters_from_earth_surface wind_flag: wind_speed_2_meters_from_earth_surface_quality_flag @@ -1158,6 +1160,12 @@ sources: description: Station identification code dtype: object long_name: wbanno + wetness: + description: This parameter indicates the presence or absence of moisture due to precipitation, in Ohms. High values (>= 1000) indicate an absence of moisture. Low values (< 1000) indicate the presence of moisture. + dtype: float32 + long_name: wetness + quality_flag: wet_flag + units: Ohms wetness_quality_flag: description: QC flag for the wetness measurement dtype: float32 @@ -1190,6 +1198,7 @@ sources: - relative_humidity - soil_moisture_5 - soil_temperature_5 + - wetness - wind_1_5 group_name: variables - columns: diff --git a/tests/conftest.py b/tests/conftest.py index 36808da..54c5374 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,7 +127,7 @@ def test_repository(test_session, test_s3_client, test_config): "insitu-observations-near-surface-temperature-us-climate-reference-network" ) service_definition = get_service_definition(dataset_name) - for dataset_source in ["USCRN_DAILY"]: + for dataset_source in ["USCRN_DAILY", "USCRN_SUBHOURLY"]: start_year, end_year = get_test_years(dataset_source) run_ingestion_pipeline( dataset_name, diff --git a/tests/test_get_forms_jsons.py b/tests/test_get_forms_jsons.py index 2bc365b..caca6fc 100644 --- a/tests/test_get_forms_jsons.py +++ b/tests/test_get_forms_jsons.py @@ -2,7 +2,9 @@ def test_get_forms_jsons(test_repository, tmp_path): - dataset = "insitu-observations-gnss" + dataset = ( + "insitu-observations-near-surface-temperature-us-climate-reference-network" + ) s3_client = test_repository.s3_client bucket = s3_client.get_bucket_name(dataset) geco_files = get_forms_jsons( From e33fc5d2138c89b0bfa346af45f416f2d7326d2b Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 19 Jul 2024 08:33:19 +0200 Subject: [PATCH 02/20] trying to fix uscrn forms --- cdsobs/constants.py | 16 +- cdsobs/data/cdsobs_config_template.yml | 16 +- .../service_definition.json | 2736 +++++++++-------- .../service_definition.yml | 988 +++--- .../service_definition_old.yml | 1235 ++++++++ tests/conftest.py | 2 +- tests/retrieve/test_adaptor.py | 8 +- tests/test_api.py | 66 +- tests/test_cdm_api.py | 4 +- tests/test_config.py | 8 +- tests/test_http_api.py | 2 +- tests/upgrade_service_definition.py | 7 +- tests/utils.py | 8 +- 13 files changed, 3252 insertions(+), 1844 deletions(-) create mode 100644 cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml diff --git a/cdsobs/constants.py b/cdsobs/constants.py index c832d59..d3d8865 100644 --- a/cdsobs/constants.py +++ b/cdsobs/constants.py @@ -65,10 +65,10 @@ IGRA_H=1981, CUON=1924, GRUAN=2010, - USCRN_SUBHOURLY=2007, - USCRN_HOURLY=2006, - USCRN_DAILY=2007, - USCRN_MONTHLY=2006, + uscrn_subhourly=2007, + uscrn_hourly=2006, + uscrn_daily=2007, + uscrn_monthly=2006, IGS=2001, EPN=1996, ) @@ -96,10 +96,10 @@ "GRUAN": ["wind_speed", "air_temperature"] }, "insitu-observations-near-surface-temperature-us-climate-reference-network": { - "USCRN_MONTHLY": ["daily_maximum_air_temperature", "accumulated_precipitation"], - "USCRN_DAILY": ["relative_humidity", "air_temperature"], - "USCRN_HOURLY": ["relative_humidity", "air_temperature"], - "USCRN_SUBHOURLY": ["relative_humidity", "air_temperature"], + "uscrn_monthly": ["daily_maximum_air_temperature", "accumulated_precipitation"], + "uscrn_daily": ["relative_humidity", "air_temperature"], + "uscrn_hourly": ["relative_humidity", "air_temperature"], + "uscrn_subhourly": ["relative_humidity", "air_temperature"], }, "insitu-observations-gnss": { "IGS": ["precipitable_water_column_era5"], diff --git a/cdsobs/data/cdsobs_config_template.yml b/cdsobs/data/cdsobs_config_template.yml index 9cfc38d..cb6cb7d 100644 --- a/cdsobs/data/cdsobs_config_template.yml +++ b/cdsobs/data/cdsobs_config_template.yml @@ -49,17 +49,17 @@ datasets: - uncertainty_table - name: insitu-observations-near-surface-temperature-us-climate-reference-network lon_tile_size: - USCRN_MONTHLY: 90 - USCRN_DAILY: 90 - USCRN_HOURLY: 90 - USCRN_SUBHOURLY: + uscrn_monthly: 90 + uscrn_daily: 90 + uscrn_hourly: 90 + uscrn_subhourly: 2006-2010: 30 2010-2030: 20 lat_tile_size: - USCRN_MONTHLY: 90 - USCRN_DAILY: 90 - USCRN_HOURLY: 90 - USCRN_SUBHOURLY: + uscrn_monthly: 90 + uscrn_daily: 90 + uscrn_hourly: 90 + uscrn_subhourly: 2006-2010: 30 2010-2030: 20 available_cdm_tables: diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json index b7cca0f..735c703 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json @@ -1,1368 +1,1374 @@ { - "products_hierarchy":[ - "variables", - "flag", - "processing_level", - "positive_total_uncertainty", - "negative_total_uncertainty", - "max_positive_total_uncertainty", - "max_negative_total_uncertainty", - "min_positive_total_uncertainty", - "min_negative_total_uncertainty", - "mean_positive_total_uncertainty", - "mean_negative_total_uncertainty", - "random_uncertainty", - "positive_random_uncertainty", - "negative_random_uncertainty", - "positive_systematic_uncertainty", - "negative_systematic_uncertainty", - "quasisystematic_uncertainty", - "positive_quasisystematic_uncertainty", - "negative_quasisystematic_uncertainty" - ], - "out_columns_order":[ - "station_name", - "alternative_name", - "report_timestamp", - "report_id", - "logbook_version", - "longitude", - "latitude", - "air_temperature", - "air_temperature_positive_random_uncertainty", - "air_temperature_positive_systematic_uncertainty", - "air_temperature_positive_quasisystematic_uncertainty", - "air_temperature_positive_total_uncertainty", - "air_temperature_negative_random_uncertainty", - "air_temperature_negative_systematic_uncertainty", - "air_temperature_negative_quasisystematic_uncertainty", - "air_temperature_negative_total_uncertainty", - "air_temperature_random_uncertainty", - "air_temperature_quasisystematic_uncertainty", - "air_temperature_max", - "air_temperature_max_negative_total_uncertainty", - "air_temperature_max_positive_total_uncertainty", - "air_temperature_min", - "air_temperature_min_negative_total_uncertainty", - "air_temperature_min_positive_total_uncertainty", - "air_temperature_mean", - "air_temperature_mean_negative_total_uncertainty", - "air_temperature_mean_positive_total_uncertainty", - "relative_humidity", - "relative_humidity_quality_flag", - "relative_humidity_max", - "relative_humidity_min", - "soil_temperature", - "soil_temperature_quality_flag", - "soil_temperature_max", - "soil_temperature_max_quality_flag", - "soil_temperature_min", - "soil_temperature_min_quality_flag", - "downward_shortwave_irradiance_at_earth_surface", - "downward_shortwave_irradiance_at_earth_surface_quality_flag", - "downward_shortwave_irradiance_at_earth_surface_max", - "downward_shortwave_irradiance_at_earth_surface_max_quality_flag", - "downward_shortwave_irradiance_at_earth_surface_min", - "downward_shortwave_irradiance_at_earth_surface_min_quality_flag", - "soil_moisture_5cm_from_earth_surface", - "soil_moisture_10cm_from_earth_surface", - "soil_moisture_20cm_from_earth_surface", - "soil_moisture_50cm_from_earth_surface", - "soil_moisture_100cm_from_earth_surface", - "soil_temperature_5cm_from_earth_surface", - "soil_temperature_10cm_from_earth_surface", - "soil_temperature_20cm_from_earth_surface", - "soil_temperature_50cm_from_earth_surface", - "soil_temperature_100cm_from_earth_surface", - "soil_temperature_processing_level", - "soil_temperature_processing_level_quality_flag", - "wind_speed_2_meters_from_earth_surface", - "wind_speed_2_meters_from_earth_surface_quality_flag", - "wetness_quality_flag", - "accumulated_precipitation", - "daily_global_solar_radiation_rates", - "monthly_global_solar_radiation_rates" - ], - "space_columns": { - "longitude": "location_longitude", - "latitude": "location_longitude" - }, - "sources":{ - "USCRN_SUBHOURLY":{ - "data_table":"unc_subhourly", - "order_by":[ - "station_name", - "report_timestamp" - ], - "space_columns": { - "longitude": "location_longitude", - "latitude": "location_longitude" - }, - "mandatory_columns":[ - "sitename", - "wbanno", - "longitude", - "latitude", - "date_of_observation" - ], - "products":[ - { - "group_name":"variables", - "columns":[ - "t", - "precipitation", - "solar_radiation", - "surface_temperature", - "relative_humidity", - "soil_moisture_5", - "soil_temperature_5", - "wind_1_5" - ] - }, - { - "group_name":"negative_total_uncertainty", - "columns":[ - "terr_m" - ] - }, - { - "group_name":"random_uncertainty", - "columns":[ - "random_pm" - ] - }, - { - "group_name":"positive_total_uncertainty", - "columns":[ - "terr_p" - ] - }, - { - "group_name":"quasisystematic_uncertainty", - "columns":[ - "quasisys_pm" - ] - }, - { - "group_name":"positive_systematic_uncertainty", - "columns":[ - "sys_p" - ] - }, - { - "group_name":"negative_systematic_uncertainty", - "columns":[ - "sys_m" - ] - }, - { - "group_name":"flag", - "columns":[ - "sr_flag", - "st_flag", - "rh_flag", - "wet_flag", - "wind_flag" - ] - }, - { - "group_name":"processing_level", - "columns":[ - "st_type" - ] - } - ], - "descriptions":{ - "id":{ - "name_for_output":"report_id", - "description":"Identifier in the USCRN meta-database" - }, - "sitename":{ - "name_for_output":"alternative_name", - "description":"Alternative name for station" - }, - "wbanno":{ - "name_for_output":"station_name", - "description":"Station identification code" - }, - "longitude":{ - "units":"degree_east", - "name_for_output":"location_longitude", - "long_name":"longitude", - "description":"Longitude of the station (deg. East)" - }, - "latitude":{ - "units":"degree_north", - "name_for_output":"location_latitude", - "long_name":"latitude", - "description":"Latitude of the station (deg. North)" - }, - "t":{ - "units":"K", - "name_for_output":"air_temperature", - "long_name":"average_air_temperature", - "description":"air temperature", - "positive_total_uncertainty":"terr_p", - "negative_total_uncertainty":"terr_m", - "random_uncertainty":"random_pm", - "quasisystematic_uncertainty":"quasisys_pm", - "negative_systematic_uncertainty":"sys_m", - "positive_systematic_uncertainty":"sys_p" - }, - "terr_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_total_uncertainty", - "long_name":"air_temperature_positive_total_uncertainty", - "description":"positive uncertainty of temperature" - }, - "terr_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_total_uncertainty", - "long_name":"air_temperature_negative_total_uncertainty", - "description":"negative uncertainty of temperature" - }, - "random_pm":{ - "units":"K", - "name_for_output":"air_temperature_random_uncertainty", - "long_name":"air_temperature_random_uncertainty", - "description":"+ve/-ve Random" - }, - "sys_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_systematic_uncertainty", - "long_name":"air_temperature_positive_systematic_uncertainty", - "description":"+ve Systematic uncertainty" - }, - "sys_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_systematic_uncertainty", - "long_name":"air_temperature_negative_systematic_uncertainty", - "description":"-ve Systematic uncertainty" - }, - "quasisys_pm":{ - "units":"K", - "name_for_output":"air_temperature_quasisystematic_uncertainty", - "long_name":"air_temperature_quasisystematic_uncertainty", - "description":"+ve/-ve Quasi-systematic" - }, - "date_of_observation":{ - "name_for_output":"report_timestamp", - "description":"observation date time UTC" - }, - "crx_vn":{ - "name_for_output":"logbook_version", - "description":"The version number of the station datalogger program that was in effect at the time of the observation" - }, - "precipitation":{ - "units":"mm", - "name_for_output":"accumulated_precipitation", - "long_name":"accumulated_precipitation", - "description":"Total amount of precipitation" - }, - "solar_radiation":{ - "units":"W m-2", - "name_for_output":"downward_shortwave_irradiance_at_earth_surface", - "long_name":"downward_shortwave_irradiance_at_earth_surface", - "description":"Average global solar radiation received", - "flag":"sr_flag" - }, - "surface_temperature":{ - "units":"K", - "name_for_output":"soil_temperature", - "long_name":"average_soil_temperature", - "description":"USCRN stations have multiple co-located soil sensors that record independent measurements. The soil values reported in this dataset are calculated from these multiple independent measurements. Soil moisture is the ratio of water volume over sample volume (m^3 water m^-3 soil).", - "flag":"st_flag", - "processing_level":"st_type" - }, - "relative_humidity":{ - "units":"%", - "name_for_output":"relative_humidity", - "long_name":"average_relative_humidity", - "description":"All USCRN stations now report 5-minute relative humidity averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22.", - "flag":"rh_flag" - }, - "soil_moisture_5":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_5cm_from_earth_surface", - "long_name":"soil_moisture_5cm_from_earth_surface", - "description":"Average soil moisture at 5 cm below the surface" - }, - "soil_temperature_5":{ - "units":"K", - "name_for_output":"soil_temperature_5cm_from_earth_surface", - "long_name":"soil_temperature_5cm_from_earth_surface", - "description":"Average soil temperature at 5 cm below the surface" - }, - "wind_1_5":{ - "units":"m s-1", - "name_for_output":"wind_speed_2_meters_from_earth_surface", - "long_name":"wind_speed_2_meters_from_earth_surface", - "description":"Average wind speed, in meters per second, at a height of 1.5 meters", - "flag":"wind_flag" - }, - "sr_flag":{ - "name_for_output":"downward_shortwave_irradiance_at_earth_surface_quality_flag", - "long_name":"downward_shortwave_irradiance_at_earth_surface_quality_flag", - "description":"QC flag for the average global solar radiation measurement" - }, - "st_flag":{ - "name_for_output":"soil_temperature_processing_level_quality_flag", - "long_name":"soil_temperature_processing_level_quality_flag", - "description":"QC flag for the surface temperature measurement" - }, - "rh_flag":{ - "name_for_output":"relative_humidity_quality_flag", - "long_name":"average_relative_humidity_quality_flag", - "description":"QC flag for the relative humidity measurement" - }, - "wet_flag":{ - "name_for_output":"wetness_quality_flag", - "long_name":"wetness_quality_flag", - "description":"QC flag for the wetness measurement" - }, - "wind_flag":{ - "name_for_output":"wind_speed_2_meters_from_earth_surface_quality_flag", - "long_name":"wind_speed_2_meters_from_earth_surface_quality_flag", - "description":"QC flag for the wind speed measurement" - }, - "st_type":{ - "name_for_output":"soil_temperature_processing_level", - "long_name":"soil_temperature_processing_level", - "description":"The type of infrared surface temperature measurement" - } - } - }, - "USCRN_HOURLY":{ - "data_table":"unc_hourly", - "order_by":[ - "station_name", - "report_timestamp" - ], - "space_columns":{ - "longitude":"location_longitude", - "latitude":"location_latitude" - }, - "mandatory_columns":[ - "sitename", - "wbanno", - "longitude", - "latitude", - "date_of_observation" - ], - "products":[ - { - "group_name":"variables", - "columns":[ - "t", - "t_max", - "t_min", - "p_calc", - "solarad", - "solarad_max", - "solarad_min", - "sur_temp", - "sur_temp_max", - "sur_temp_min", - "rh_hr_avg", - "soil_moisture_5", - "soil_moisture_10", - "soil_moisture_20", - "soil_moisture_50", - "soil_moisture_100", - "soil_temp_5", - "soil_temp_10", - "soil_temp_20", - "soil_temp_50", - "soil_temp_100" - ] - }, - { - "group_name":"random_uncertainty", - "columns":[ - "random_pm" - ] - }, - { - "group_name":"positive_total_uncertainty", - "columns":[ - "terr_p" - ] - }, - { - "group_name":"negative_total_uncertainty", - "columns":[ - "terr_m" - ] - }, - { - "group_name":"max_positive_total_uncertainty", - "columns":[ - "tmaxerr_p" - ] - }, - { - "group_name":"max_negative_total_uncertainty", - "columns":[ - "tmaxerr_m" - ] - }, - { - "group_name":"min_positive_total_uncertainty", - "columns":[ - "tminerr_p" - ] - }, - { - "group_name":"min_negative_total_uncertainty", - "columns":[ - "tminerr_m" - ] - }, - { - "group_name":"positive_systematic_uncertainty", - "columns":[ - "sys_p" - ] - }, - { - "group_name":"negative_systematic_uncertainty", - "columns":[ - "sys_m" - ] - }, - { - "group_name":"positive_quasisystematic_uncertainty", - "columns":[ - "quasisys_p" - ] - }, - { - "group_name":"negative_quasisystematic_uncertainty", - "columns":[ - "quasisys_m" - ] - }, - { - "group_name":"flag", - "columns":[ - "solarad_flag", - "solarad_max_flag", - "solarad_min_flag", - "sur_temp_flag", - "sur_temp_max_flag", - "sur_temp_min_flag", - "rh_hr_avg_flag" - ] - }, - { - "group_name":"processing_level", - "columns":[ - "sur_temp_type" - ] - } - ], - "descriptions":{ - "id":{ - "name_for_output":"report_id", - "description":"Identifier in the USCRN meta-database" - }, - "sitename":{ - "name_for_output":"alternative_name", - "description":"Alternative name for station" - }, - "wbanno":{ - "name_for_output":"station_name", - "description":"Station identification code" - }, - "longitude":{ - "units":"degree_east", - "name_for_output":"location_longitude", - "long_name":"longitude", - "description":"Longitude of the station (deg. East)" - }, - "latitude":{ - "units":"degree_north", - "name_for_output":"location_latitude", - "long_name":"latitude", - "description":"Latitude of the station (deg. North)" - }, - "t":{ - "units":"K", - "name_for_output":"air_temperature", - "long_name":"average_air_temperature", - "description":"Near surface temperature", - "positive_total_uncertainty":"terr_p", - "negative_total_uncertainty":"terr_m", - "max_positive_total_uncertainty":"tmaxerr_p", - "max_negative_total_uncertainty":"tmaxerr_m", - "min_positive_total_uncertainty":"tminerr_p", - "min_negative_total_uncertainty":"tminerr_m", - "random_uncertainty":"random_pm", - "positive_systematic_uncertainty":"sys_p", - "negative_systematic_uncertainty":"sys_m", - "positive_quasisystematic_uncertainty":"quasisys_p", - "negative_quasisystematic_uncertainty":"quasisys_m" - }, - "terr_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_total_uncertainty", - "long_name":"air_temperature_positive_total_uncertainty", - "description":"positive uncertainty of temperature" - }, - "terr_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_total_uncertainty", - "long_name":"air_temperature_negative_total_uncertainty", - "description":"negative uncertainty of temperature" - }, - "random_pm":{ - "units":"K", - "name_for_output":"air_temperature_random_uncertainty", - "long_name":"air_temperature_random_uncertainty", - "description":"+ve/-ve Random" - }, - "sys_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_systematic_uncertainty", - "long_name":"air_temperature_positive_systematic_uncertainty", - "description":"+ve Systematic uncertainty" - }, - "sys_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_systematic_uncertainty", - "long_name":"air_temperature_negative_systematic_uncertainty", - "description":"-ve Systematic uncertainty" - }, - "tmaxerr_p":{ - "units":"K", - "name_for_output":"air_temperature_max_positive_total_uncertainty", - "long_name":"air_temperature_max_positive_total_uncertainty", - "description":"positive uncertainties in tmax which are related to temperature" - }, - "tmaxerr_m":{ - "units":"K", - "name_for_output":"air_temperature_max_negative_total_uncertainty", - "long_name":"air_temperature_max_negative_total_uncertainty", - "description":"negative uncertainties in tmax which are related to temperature" - }, - "tminerr_p":{ - "units":"K", - "name_for_output":"air_temperature_min_positive_total_uncertainty", - "long_name":"minimum_air_temperature_positive_total_uncertainty", - "description":"positive uncertainties in tmin which are related to temperature" - }, - "tminerr_m":{ - "units":"K", - "name_for_output":"air_temperature_min_negative_total_uncertainty", - "long_name":"minimum_air_temperature_negative_total_uncertainty", - "description":"negative uncertainties in tmin which are related to temperature" - }, - "quasisys_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_quasisystematic_uncertainty", - "long_name":"air_temperature_positive_quasisystematic_uncertainty", - "description":"+ve Quasi-systematic uncertainty" - }, - "quasisys_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_quasisystematic_uncertainty", - "long_name":"air_temperature_negative_quasisystematic_uncertainty", - "description":"-ve/-ve Quasi-systematic" - }, - "date_of_observation":{ - "name_for_output":"report_timestamp", - "description":"observation date time UTC" - }, - "crx_vn":{ - "name_for_output":"logbook_version", - "description":"The version number of the station datalogger program that was in effect at the time of the observation" - }, - "t_max":{ - "units":"K", - "name_for_output":"daily_maximum_air_temperature", - "long_name":"air_temperature_max", - "description":"Maximum air temperature during the hour" - }, - "t_min":{ - "units":"K", - "name_for_output":"daily_minimum_air_temperature", - "long_name":"minimum_air_temperature", - "description":"Minimum air temperature during the hour" - }, - "p_calc":{ - "units":"mm", - "name_for_output":"accumulated_precipitation", - "long_name":"accumulated_precipitation", - "description":"Total amount of precipitation, in mm, recorded during the hour" - }, - "solarad":{ - "units":"W m-2", - "name_for_output":"downward_shortwave_irradiance_at_earth_surface", - "long_name":"downward_shortwave_irradiance_at_earth_surface", - "description":"Average global solar radiation", - "flag":"solarad_flag" - }, - "solarad_flag":{ - "name_for_output":"downward_shortwave_irradiance_at_earth_surface_quality_flag", - "long_name":"downward_shortwave_irradiance_at_earth_surface", - "description":"QC flag for average global solar radiation. 0 denotes good data and 3 denotes erroneous data." - }, - "solarad_max":{ - "units":"W m-2", - "name_for_output":"downward_shortwave_irradiance_at_earth_surface_max", - "long_name":"maximum_downward_shortwave_irradiance_at_earth_surface", - "description":"Maximum global solar radiation", - "flag":"solarad_max_flag" - }, - "solarad_max_flag":{ - "name_for_output":"downward_shortwave_irradiance_at_earth_surface_max_quality_flag", - "long_name":"maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "description":"QC flag for maximum global solar radiation. 0 denotes good data and 3 denotes erroneous data." - }, - "solarad_min":{ - "units":"W m-2", - "name_for_output":"downward_shortwave_irradiance_at_earth_surface_min", - "long_name":"minimum_downward_shortwave_irradiance_at_earth_surface", - "description":"Minimum global solar radiation", - "flag":"solarad_min_flag" - }, - "solarad_min_flag":{ - "name_for_output":"downward_shortwave_irradiance_at_earth_surface_min_quality_flag", - "long_name":"minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "description":"QC flag for minimum global solar radiation. 0 denotes good data and 3 denotes erroneous data." - }, - "sur_temp":{ - "units":"K", - "name_for_output":"soil_temperature", - "long_name":"average_soil_temperature", - "description":"On 2013-01-07 at 1500 UTC, USCRN began reporting corrected surface temperature measurements for some stations. These changes impact previous users of the data because the corrected values differ from uncorrected values. To distinguish between uncorrected (raw) and corrected surface temperature measurements, a surface temperature type field was added to the data product. The possible values of the this field are \"R\" to denote raw surface temperature measurements, \"C\" to denote corrected surface temperature measurements, and \"U\" for unknown/missing.", - "flag":"sur_temp_flag", - "processing_level":"sur_temp_type" - }, - "sur_temp_flag":{ - "name_for_output":"soil_temperature_quality_flag", - "long_name":"average_soil_temperature_quality_flag", - "description":"QC flag for infrared surface temperature. 0 denotes good data and 3 denotes erroneous data." - }, - "sur_temp_type":{ - "name_for_output":"soil_temperature_processing_level", - "long_name":"soil_temperature_processing_level", - "description":"Type of infrared surface temperature measurement" - }, - "sur_temp_max":{ - "units":"K", - "name_for_output":"soil_temperature_max", - "long_name":"maximum_soil_temperature", - "description":"Maximum infrared surface temperature", - "flag":"sur_temp_max_flag" - }, - "sur_temp_max_flag":{ - "name_for_output":"soil_temperature_max_quality_flag", - "long_name":"maximum_soil_temperature_quality_flag", - "description":"QC flag for infrared surface temperature maximum. 0 denotes good data and 3 denotes erroneous data." - }, - "sur_temp_min":{ - "units":"K", - "name_for_output":"soil_temperature_min", - "long_name":"minimum_soil_temperature", - "description":"Minimum infrared surface temperature", - "flag":"sur_temp_min_flag" - }, - "sur_temp_min_flag":{ - "name_for_output":"soil_temperature_min_quality_flag", - "long_name":"minimum_soil_temperature_quality_flag", - "description":"QC flag for infrared surface temperature minimum. 0 denotes good data and 3 denotes erroneous data." - }, - "rh_hr_avg":{ - "units":"%", - "name_for_output":"relative_humidity", - "long_name":"average_relative_humidity", - "description":"Hourly relative humidity is computed from 5-minute averages in almost all cases, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22.", - "flag":"rh_hr_avg_flag" - }, - "rh_hr_avg_flag":{ - "name_for_output":"relative_humidity_quality_flag", - "long_name":"average_relative_humidity_quality_flag", - "description":"QC flag for RH average. 0 denotes good data and 3 denotes erroneous data." - }, - "soil_moisture_5":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_5cm_from_earth_surface", - "long_name":"soil_moisture_5cm_from_earth_surface", - "description":"Average soil moisture at 5 cm below the surface" - }, - "soil_moisture_10":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_10cm_from_earth_surface", - "long_name":"soil_moisture_10cm_from_earth_surface", - "description":"Average soil moisture at 10 cm below the surface" - }, - "soil_moisture_20":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_20cm_from_earth_surface", - "long_name":"soil_moisture_20cm_from_earth_surface", - "description":"Average soil moisture at 20 cm below the surface" - }, - "soil_moisture_50":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_50cm_from_earth_surface", - "long_name":"soil_moisture_50cm_from_earth_surface", - "description":"Average soil moisture at 50 cm below the surface" - }, - "soil_moisture_100":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_100cm_from_earth_surface", - "long_name":"soil_moisture_100cm_from_earth_surface", - "description":"Average soil moisture at 100 cm below the surface" - }, - "soil_temp_5":{ - "units":"K", - "name_for_output":"soil_temperature_5cm_from_earth_surface", - "long_name":"soil_temperature_5cm_from_earth_surface", - "description":"Average soil temperature at 5 cm below the surface" - }, - "soil_temp_10":{ - "units":"K", - "name_for_output":"soil_temperature_10cm_from_earth_surface", - "long_name":"soil_temperature_10cm_from_earth_surface", - "description":"Average soil temperature at 10 cm below the surface" - }, - "soil_temp_20":{ - "units":"K", - "name_for_output":"soil_temperature_20cm_from_earth_surface", - "long_name":"soil_temperature_20cm_from_earth_surface", - "description":"Average soil temperature at 20 cm below the surface" - }, - "soil_temp_50":{ - "units":"K", - "name_for_output":"soil_temperature_50cm_from_earth_surface", - "long_name":"soil_temperature_50cm_from_earth_surface", - "description":"Average soil temperature at 50 cm below the surface" - }, - "soil_temp_100":{ - "units":"K", - "name_for_output":"soil_temperature_100cm_from_earth_surface", - "long_name":"soil_temperature_100cm_from_earth_surface", - "description":"Average soil temperature at 100 cm below the surface" - } - } - }, - "USCRN_DAILY":{ - "data_table":"unc_daily", - "order_by":[ - "station_name", - "report_timestamp" - ], - "space_columns":{ - "longitude":"location_longitude", - "latitude":"location_latitude" - }, - "mandatory_columns":[ - "sitename", - "wbanno", - "longitude", - "latitude", - "date_of_observation" - ], - "products":[ - { - "group_name":"variables", - "columns":[ - "t", - "t_daily_max", - "t_daily_min", - "t_daily_mean", - "p_daily_calc", - "sur_temp_daily_avg", - "rh_daily_max", - "rh_daily_min", - "rh_daily_avg", - "soil_moisture_5_daily", - "soil_moisture_10_daily", - "soil_moisture_20_daily", - "soil_moisture_50_daily", - "soil_moisture_100_daily", - "soil_temp_5_daily", - "soil_temp_10_daily", - "soil_temp_20_daily", - "soil_temp_50_daily", - "soil_temp_100_daily" - ] - }, - { - "group_name":"positive_random_uncertainty", - "columns":[ - "random_p" - ] - }, - { - "group_name":"negative_random_uncertainty", - "columns":[ - "random_m" - ] - }, - { - "group_name":"positive_total_uncertainty", - "columns":[ - "terr_p", - "tmaxerr_p", - "tminerr_p", - "tmeanerr_p" - ] - }, - { - "group_name":"negative_total_uncertainty", - "columns":[ - "terr_m", - "tmaxerr_m", - "tminerr_m", - "tmeanerr_m" - ] - }, - { - "group_name":"positive_quasisystematic_uncertainty", - "columns":[ - "quasisys_p" - ] - }, - { - "group_name":"negative_quasisystematic_uncertainty", - "columns":[ - "quasisys_m" - ] - }, - { - "group_name":"positive_systematic_uncertainty", - "columns":[ - "sys_p" - ] - }, - { - "group_name":"negative_systematic_uncertainty", - "columns":[ - "sys_m" - ] - }, - { - "group_name":"processing_level", - "columns":[ - "sur_temp_daily_type" - ] - } - ], - "descriptions":{ - "id":{ - "name_for_output":"report_id", - "description":"Identifier in the USCRN meta-database" - }, - "sitename":{ - "name_for_output":"alternative_name", - "description":"Alternative name for station" - }, - "wbanno":{ - "name_for_output":"station_name", - "description":"Station identification code" - }, - "longitude":{ - "units":"degree_east", - "name_for_output":"location_longitude", - "long_name":"longitude", - "description":"Longitude of the station (deg. East)" - }, - "latitude":{ - "units":"degree_north", - "name_for_output":"location_latitude", - "long_name":"latitude", - "description":"Latitude of the station (deg. North)" - }, - "t":{ - "units":"K", - "name_for_output":"air_temperature", - "long_name":"average_air_temperature", - "description":"Near surface temperature", - "positive_systematic_uncertainty":"sys_p", - "negative_systematic_uncertainty":"sys_m", - "positive_total_uncertainty":"terr_p", - "negative_total_uncertainty":"terr_m", - "positive_random_uncertainty":"random_p", - "negative_random_uncertainty":"random_m" - }, - "terr_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_total_uncertainty", - "long_name":"air_temperature_positive_total_uncertainty", - "description":"positive uncertainty of temperature" - }, - "terr_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_total_uncertainty", - "long_name":"air_temperature_negative_total_uncertainty", - "description":"negative uncertainty of temperature" - }, - "tmaxerr_p":{ - "units":"K", - "name_for_output":"air_temperature_max_positive_total_uncertainty", - "long_name":"air_temperature_max_positive_total_uncertainty", - "description":"positive uncertainties in tmax which are related to temperature" - }, - "tmaxerr_m":{ - "units":"K", - "name_for_output":"air_temperature_max_negative_total_uncertainty", - "long_name":"air_temperature_max_negative_total_uncertainty", - "description":"negative uncertainties in tmax which are related to temperature" - }, - "tminerr_p":{ - "units":"K", - "name_for_output":"air_temperature_min_positive_total_uncertainty", - "long_name":"minimum_air_temperature_positive_total_uncertainty", - "description":"positive uncertainties in tmin which are related to temperature" - }, - "tminerr_m":{ - "units":"K", - "name_for_output":"air_temperature_min_negative_total_uncertainty", - "long_name":"minimum_air_temperature_negative_total_uncertainty", - "description":"negative uncertainties in tmin which are related to temperature" - }, - "tmeanerr_p":{ - "units":"K", - "name_for_output":"air_temperature_mean_positive_total_uncertainty", - "long_name":"mean_air_temperature_positive_total_uncertainty", - "description":"tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (positive part using the Hourly data)" - }, - "tmeanerr_m":{ - "units":"K", - "name_for_output":"air_temperature_mean_negative_total_uncertainty", - "long_name":"mean_air_temperature_negative_total_uncertainty", - "description":"tmeanerr_m, tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (negative per using the Hourly data)" - }, - "random_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_random_uncertainty", - "long_name":"air_temperature_positive_random_uncertainty", - "description":"+ve Random uncertainty" - }, - "random_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_random_uncertainty", - "long_name":"air_temperature_negative_random_uncertainty", - "description":"-ve Random uncertainty" - }, - "sys_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_systematic_uncertainty", - "long_name":"air_temperature_positive_systematic_uncertainty", - "description":"+ve Systematic uncertainty" - }, - "sys_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_systematic_uncertainty", - "long_name":"air_temperature_negative_systematic_uncertainty", - "description":"-ve Systematic uncertainty" - }, - "quasisys_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_quasisystematic_uncertainty", - "long_name":"air_temperature_positive_quasisystematic_uncertainty", - "description":"+ve Quasi-systematic uncertainty" - }, - "quasisys_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_quasisystematic_uncertainty", - "long_name":"air_temperature_negative_quasisystematic_uncertainty", - "description":"-ve/-ve Quasi-systematic" - }, - "date_of_observation":{ - "name_for_output":"report_timestamp", - "description":"observation date time UTC" - }, - "crx_vn":{ - "name_for_output":"logbook_version", - "description":"The version number of the station datalogger program that was in effect at the time of the observation" - }, - "t_daily_max":{ - "units":"K", - "name_for_output":"daily_maximum_air_temperature", - "long_name":"air_temperature_max", - "description":"Maximum air temperature", - "positive_total_uncertainty":"tmaxerr_p", - "negative_total_uncertainty":"tmaxerr_m" - }, - "t_daily_min":{ - "units":"K", - "name_for_output":"daily_minimum_air_temperature", - "long_name":"minimum_air_temperature", - "description":"Minimum air temperature", - "positive_total_uncertainty":"tminerr_p", - "negative_total_uncertainty":"tminerr_m" - }, - "t_daily_mean":{ - "units":"K", - "name_for_output":"air_temperature", - "long_name":"mean_air_temperature", - "description":"Mean air temperature calculated using the typical historical approach: (T_DAILY_MAX + T_DAILY_MIN) / 2", - "positive_total_uncertainty":"tmeanerr_p", - "negative_total_uncertainty":"tmeanerr_m" - }, - "p_daily_calc":{ - "units":"mm", - "name_for_output":"accumulated_precipitation", - "long_name":"accumulated_precipitation", - "description":"Total amount of precipitation" - }, - "sur_temp_daily_type":{ - "name_for_output":"soil_temperature_processing_level", - "long_name":"soil_temperature_processing_level", - "description":"Type of infrared surface temperature measurement." - }, - "sur_temp_daily_avg":{ - "units":"K", - "name_for_output":"soil_temperature", - "long_name":"average_soil_temperature", - "description":"Average infrared surface temperature", - "processing_level":"sur_temp_daily_type" - }, - "rh_daily_max":{ - "units":"%", - "name_for_output":"relative_humidity_max", - "long_name":"maximum_relative_humidity", - "description":"Maximum relative humidity" - }, - "rh_daily_min":{ - "units":"%", - "name_for_output":"relative_humidity_min", - "long_name":"minimum_relative_humidity", - "description":"Minimum relative humidity" - }, - "rh_daily_avg":{ - "units":"%", - "name_for_output":"relative_humidity", - "long_name":"average_relative_humidity", - "description":"Relative humidity is computed from 5-minute values in almost all cases. All USCRN stations now report 5-minute averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22." - }, - "soil_moisture_5_daily":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_5cm_from_earth_surface", - "long_name":"soil_moisture_5cm_from_earth_surface", - "description":"Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 5 cm below the surface." - }, - "soil_moisture_10_daily":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_10cm_from_earth_surface", - "long_name":"soil_moisture_10cm_from_earth_surface", - "description":"Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 10 cm below the surface." - }, - "soil_moisture_20_daily":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_20cm_from_earth_surface", - "long_name":"soil_moisture_20cm_from_earth_surface", - "description":"Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 20 cm below the surface." - }, - "soil_moisture_50_daily":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_50cm_from_earth_surface", - "long_name":"soil_moisture_50cm_from_earth_surface", - "description":"Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 50 cm below the surface." - }, - "soil_moisture_100_daily":{ - "units":"m3 m-3", - "name_for_output":"soil_moisture_100cm_from_earth_surface", - "long_name":"soil_moisture_100cm_from_earth_surface", - "description":"Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 100 cm below the surface." - }, - "soil_temp_5_daily":{ - "units":"K", - "name_for_output":"soil_temperature_5cm_from_earth_surface", - "long_name":"soil_temperature_5cm_from_earth_surface", - "description":"Average soil temperature, in degrees C, at 5 cm below the surface." - }, - "soil_temp_10_daily":{ - "units":"K", - "name_for_output":"soil_temperature_10cm_from_earth_surface", - "long_name":"soil_temperature_10cm_from_earth_surface", - "description":"Average soil temperature, in degrees C, at 10 cm below the surface." - }, - "soil_temp_20_daily":{ - "units":"K", - "name_for_output":"soil_temperature_20cm_from_earth_surface", - "long_name":"soil_temperature_20cm_from_earth_surface", - "description":"Average soil temperature, in degrees C, at 20 cm below the surface." - }, - "soil_temp_50_daily":{ - "units":"K", - "name_for_output":"soil_temperature_50cm_from_earth_surface", - "long_name":"soil_temperature_50cm_from_earth_surface", - "description":"Average soil temperature, in degrees C, at 50 cm below the surface." - }, - "soil_temp_100_daily":{ - "units":"K", - "name_for_output":"soil_temperature_100cm_from_earth_surface", - "long_name":"soil_temperature_100cm_from_earth_surface", - "description":"Average soil temperature, in degrees C, at 100 cm below the surface." - } - } - }, - "USCRN_MONTHLY":{ - "data_table":"unc_monthly", - "order_by":[ - "station_name", - "report_timestamp" - ], - "space_columns":{ - "longitude":"location_longitude", - "latitude":"location_latitude" - }, - "mandatory_columns":[ - "sitename", - "wbanno", - "longitude", - "latitude", - "date_of_observation" - ], - "products":[ - { - "group_name":"variables", - "columns":[ - "t", - "t_monthly_max", - "t_monthly_min", - "t_monthly_mean", - "p_monthly_calc", - "sur_temp_monthly_max", - "sur_temp_monthly_min", - "sur_temp_monthly_avg" - ] - }, - { - "group_name":"positive_random_uncertainty", - "columns":[ - "random_p" - ] - }, - { - "group_name":"negative_random_uncertainty", - "columns":[ - "random_m" - ] - }, - { - "group_name":"positive_total_uncertainty", - "columns":[ - "terr_p", - "tmaxerr_p", - "tminerr_p" - ] - }, - { - "group_name":"negative_total_uncertainty", - "columns":[ - "terr_m", - "tmaxerr_m", - "tminerr_m" - ] - }, - { - "group_name":"positive_quasisystematic_uncertainty", - "columns":[ - "quasisys_p" - ] - }, - { - "group_name":"negative_quasisystematic_uncertainty", - "columns":[ - "quasisys_m" - ] - }, - { - "group_name":"positive_systematic_uncertainty", - "columns":[ - "sys_p" - ] - }, - { - "group_name":"negative_systematic_uncertainty", - "columns":[ - "sys_m" - ] - }, - { - "group_name":"processing_level", - "columns":[ - "sur_temp_monthly_type" - ] - } - ], - "descriptions":{ - "id":{ - "name_for_output":"report_id", - "description":"Identifier in the USCRN meta-database" - }, - "sitename":{ - "name_for_output":"alternative_name", - "description":"Alternative name for station" - }, - "wbanno":{ - "name_for_output":"station_name", - "description":"Station identification code" - }, - "t":{ - "units":"K", - "name_for_output":"air_temperature", - "long_name":"average_air_temperature", - "description":"Air temperature", - "positive_systematic_uncertainty":"sys_p", - "negative_systematic_uncertainty":"sys_m", - "positive_total_uncertainty":"terr_p", - "negative_total_uncertainty":"terr_m", - "positive_random_uncertainty":"random_p", - "negative_random_uncertainty":"random_m", - "positive_quasisystematic_uncertainty":"quasisys_p", - "negative_quasisystematic_uncertainty":"quasisys_n" - }, - "terr_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_total_uncertainty", - "long_name":"air_temperature_positive_total_uncertainty", - "description":"positive uncertainty of temperature" - }, - "terr_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_total_uncertainty", - "long_name":"air_temperature_negative_total_uncertainty", - "description":"negative uncertainty of temperature" - }, - "tmaxerr_p":{ - "units":"K", - "name_for_output":"air_temperature_max_positive_total_uncertainty", - "long_name":"air_temperature_max_positive_total_uncertainty", - "description":"positive uncertainties in tmax which are related to temperature" - }, - "tmaxerr_m":{ - "units":"K", - "name_for_output":"air_temperature_max_negative_total_uncertainty", - "long_name":"air_temperature_max_negative_total_uncertainty", - "description":"negative uncertainties in tmax which are related to temperature" - }, - "tminerr_p":{ - "units":"K", - "name_for_output":"air_temperature_min_positive_total_uncertainty", - "long_name":"minimum_air_temperature_positive_total_uncertainty", - "description":"positive uncertainties in tmin which are related to temperature" - }, - "tminerr_m":{ - "units":"K", - "name_for_output":"air_temperature_min_negative_total_uncertainty", - "long_name":"minimum_air_temperature_negative_total_uncertainty", - "description":"negative uncertainties in tmin which are related to temperature" - }, - "random_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_random_uncertainty", - "long_name":"air_temperature_positive_random_uncertainty", - "description":"+ve Random uncertainty" - }, - "random_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_random_uncertainty", - "long_name":"air_temperature_negative_random_uncertainty", - "description":"-ve Random uncertainty" - }, - "sys_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_systematic_uncertainty", - "long_name":"air_temperature_positive_systematic_uncertainty", - "description":"+ve Systematic uncertainty" - }, - "sys_m":{ - "units":"K", - "name_for_output":"air_temperature_negative_systematic_uncertainty", - "long_name":"air_temperature_negative_systematic_uncertainty", - "description":"-ve Systematic uncertainty" - }, - "quasisys_p":{ - "units":"K", - "name_for_output":"air_temperature_positive_quasisystematic_uncertainty", - "long_name":"air_temperature_positive_quasisystematic_uncertainty", - "description":"+ve Quasi-systematic uncertainty" - }, - "quasisys_n":{ - "units":"K", - "name_for_output":"air_temperature_negative_quasisystematic_uncertainty", - "long_name":"air_temperature_negative_quasisystematic_uncertainty", - "description":"-ve/-ve Quasi-systematic" - }, - "date_of_observation":{ - "name_for_output":"report_timestamp", - "description":"observation date time UTC" - }, - "crx_vn_monthly":{ - "name_for_output":"logbook_version", - "description":"The version number of the station datalogger program that was in effect at the time of the observation" - }, - "longitude":{ - "units":"degree_east", - "name_for_output":"location_longitude", - "long_name":"longitude", - "description":"Station longitude, using WGS-84, with a precision of 4 decimal places" - }, - "latitude":{ - "units":"degree_north", - "name_for_output":"location_latitude", - "long_name":"latitude", - "description":"Station latitude, using WGS-84, with a precision of 4 decimal places" - }, - "t_monthly_max":{ - "units":"K", - "name_for_output":"daily_maximum_air_temperature", - "long_name":"air_temperature_max", - "description":"The maximum air temperature", - "positive_total_uncertainty":"tmaxerr_p", - "negative_total_uncertainty":"tmaxerr_m" - }, - "t_monthly_min":{ - "units":"K", - "name_for_output":"daily_minimum_air_temperature", - "long_name":"minimum_air_temperature", - "description":"The minimum air temperature", - "positive_total_uncertainty":"tminerr_p", - "negative_total_uncertainty":"tminerr_m" - }, - "t_monthly_mean":{ - "units":"K", - "name_for_output":"air_temperature", - "long_name":"mean_air_temperature", - "description":"The mean air temperature calculated using the typical historical approach of (T_MONTHLY_MAX + T_MONTHLY_MIN) / 2" - }, - "p_monthly_calc":{ - "units":"mm", - "name_for_output":"accumulated_precipitation", - "long_name":"accumulated_precipitation", - "description":"The total amount of precipitation" - }, - "sur_temp_monthly_type":{ - "name_for_output":"soil_temperature_processing_level", - "long_name":"soil_temperature_processing_level", - "description":"Type of infrared surface temperature measurement" - }, - "sur_temp_monthly_max":{ - "units":"K", - "name_for_output":"soil_temperature_max", - "long_name":"maximum_soil_temperature", - "description":"The maximum infrared surface temperature" - }, - "sur_temp_monthly_min":{ - "units":"K", - "name_for_output":"soil_temperature_min", - "long_name":"minimum_soil_temperature", - "description":"The minimum infrared surface temperature" - }, - "sur_temp_monthly_avg":{ - "units":"K", - "name_for_output":"soil_temperature", - "long_name":"average_soil_temperature", - "description":"Monthly maximum/minimum/average surface temperatures are the average of all available daily max/min/avg values. To be considered valid, there must be fewer than 4 consecutive values missing, and no more than 5 total values missing.", - "processing_level":"sur_temp_monthly_type" - } - } + "products_hierarchy": [ + "variables", + "quality_flag", + "processing_level", + "positive_total_uncertainty", + "negative_total_uncertainty", + "random_uncertainty", + "positive_random_uncertainty", + "negative_random_uncertainty", + "positive_systematic_uncertainty", + "negative_systematic_uncertainty", + "positive_quasisystematic_uncertainty", + "negative_quasisystematic_uncertainty" + ], + "out_columns_order": [ + "station_name", + "alternative_name", + "report_timestamp", + "report_id", + "logbook_version", + "longitude", + "latitude", + "air_temperature", + "air_temperature_positive_random_uncertainty", + "air_temperature_positive_systematic_uncertainty", + "air_temperature_positive_quasisystematic_uncertainty", + "air_temperature_positive_total_uncertainty", + "air_temperature_negative_random_uncertainty", + "air_temperature_negative_systematic_uncertainty", + "air_temperature_negative_quasisystematic_uncertainty", + "air_temperature_negative_total_uncertainty", + "air_temperature_random_uncertainty", + "maximum_air_temperature", + "maximum_air_temperature_negative_total_uncertainty", + "maximum_air_temperature_positive_total_uncertainty", + "minimum_air_temperature", + "minimum_air_temperature_negative_total_uncertainty", + "minimum_air_temperature_positive_total_uncertainty", + "mean_air_temperature", + "mean_air_temperature_negative_total_uncertainty", + "mean_air_temperature_positive_total_uncertainty", + "relative_humidity", + "relative_humidity_quality_flag", + "maximum_relative_humidity", + "minimum_relative_humidity", + "soil_temperature", + "soil_temperature_quality_flag", + "maximum_soil_temperature", + "maximum_soil_temperature_quality_flag", + "minimum_soil_temperature", + "minimum_soil_temperature_quality_flag", + "solar_irradiance", + "solar_irradiance_quality_flag", + "maximum_solar_irradiance", + "maximum_solar_irradiance_quality_flag", + "minimum_solar_irradiance", + "minimum_solar_irradiance_quality_flag", + "soil_moisture_5cm_from_earth_surface", + "soil_moisture_10cm_from_earth_surface", + "soil_moisture_20cm_from_earth_surface", + "soil_moisture_50cm_from_earth_surface", + "soil_moisture_100cm_from_earth_surface", + "soil_temperature_5cm_from_earth_surface", + "soil_temperature_10cm_from_earth_surface", + "soil_temperature_20cm_from_earth_surface", + "soil_temperature_50cm_from_earth_surface", + "soil_temperature_100cm_from_earth_surface", + "soil_temperature_processing_level", + "soil_temperature_processing_level_quality_flag", + "2m_wind_speed", + "2m_wind_speed_quality_flag", + "wetness", + "wetness_quality_flag", + "accumulated_precipitation", + "daily_global_solar_radiation", + "monthly_global_solar_radiation" + ], + "space_columns": { + "longitude": "longitude", + "latitude": "latitude" + }, + "sources": { + "uscrn_subhourly": { + "data_table": "unc_subhourly", + "order_by": [ + "station_name", + "report_timestamp" + ], + "mandatory_columns": [ + "sitename", + "wbanno", + "longitude", + "latitude", + "date_of_observation" + ], + "products": [ + { + "group_name": "variables", + "columns": [ + "t", + "precipitation", + "solar_radiation", + "surface_temperature", + "relative_humidity", + "soil_moisture_5", + "soil_temperature_5", + "wetness", + "wind_1_5" + ] + }, + { + "group_name": "negative_total_uncertainty", + "columns": [ + "terr_m" + ] + }, + { + "group_name": "random_uncertainty", + "columns": [ + "random_pm" + ] + }, + { + "group_name": "positive_total_uncertainty", + "columns": [ + "terr_p" + ] + }, + { + "group_name": "negative_total_uncertainty", + "columns": [ + "terr_m" + ] + }, + { + "group_name": "positive_systematic_uncertainty", + "columns": [ + "sys_p" + ] + }, + { + "group_name": "negative_systematic_uncertainty", + "columns": [ + "sys_m" + ] + }, + { + "group_name": "quality_flag", + "columns": [ + "sr_flag", + "st_flag", + "rh_flag", + "wet_flag", + "wind_flag" + ] + }, + { + "group_name": "processing_level", + "columns": [ + "st_type" + ] + } + ], + "descriptions": { + "id": { + "name_for_output": "report_id", + "description": "Link to header information." + }, + "sitename": { + "name_for_output": "alternative_name", + "description": "This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station \"VA_Charlottesville_2_SSE\" refers to a station located 2 miles South-South-East of Charlottesville, Virginia." + }, + "wbanno": { + "name_for_output": "station_name", + "description": "This is the station identification code." + }, + "longitude": { + "units": "degree_east", + "name_for_output": "longitude", + "long_name": "longitude", + "description": "Longitude of the measurement station, -180.0 to 180.0." + }, + "latitude": { + "units": "degree_north", + "name_for_output": "latitude", + "long_name": "latitude", + "description": "Latitude of the measurement station, -90 to 90." + }, + "t": { + "units": "K", + "name_for_output": "air_temperature", + "long_name": "average_air_temperature", + "description": "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties.", + "positive_total_uncertainty": "terr_p", + "negative_total_uncertainty": "terr_m", + "random_uncertainty": "random_pm", + "negative_systematic_uncertainty": "sys_m", + "positive_systematic_uncertainty": "sys_p" + }, + "terr_p": { + "units": "K", + "name_for_output": "air_temperature_positive_total_uncertainty", + "long_name": "air_temperature_positive_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "terr_m": { + "units": "K", + "name_for_output": "air_temperature_negative_total_uncertainty", + "long_name": "air_temperature_negative_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "random_pm": { + "units": "K", + "name_for_output": "air_temperature_random_uncertainty", + "long_name": "air_temperature_random_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_p": { + "units": "K", + "name_for_output": "air_temperature_positive_systematic_uncertainty", + "long_name": "air_temperature_positive_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_m": { + "units": "K", + "name_for_output": "air_temperature_negative_systematic_uncertainty", + "long_name": "air_temperature_negative_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "date_of_observation": { + "name_for_output": "report_timestamp", + "description": "This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0)." + }, + "crx_vn": { + "name_for_output": "logbook_version", + "description": "LogBook software and version." + }, + "precipitation": { + "units": "mm", + "name_for_output": "accumulated_precipitation", + "long_name": "accumulated_precipitation", + "description": "This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area." + }, + "solar_radiation": { + "units": "W m-2", + "name_for_output": "solar_irradiance", + "long_name": "solar_irradiance", + "description": "The quantity with standard name solar_irradiance, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. \"Irradiance\" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy.", + "quality_flag": "sr_flag" + }, + "surface_temperature": { + "units": "K", + "name_for_output": "soil_temperature", + "long_name": "soil_temperature", + "description": "Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period.", + "quality_flag": "st_flag", + "processing_level": "st_type" + }, + "relative_humidity": { + "units": "%", + "name_for_output": "relative_humidity", + "long_name": "relative_humidity", + "description": "This parameter is the ratio of the amount of atmospheric moisture present in the air relative to the amount that would be present if the air were saturated with respect to water or ice. This ratio is multiplied by 100 to provide numbers in percent (%). The measurements are taken at a height of 2 metres above the ground surface. The instrument is a capacitive thin-film device. It measures the flow of electricity across two electrodes separated by a polymer film. The capacitance of the film is converted to relative humidity by a calibration equation. The values are averaged over 5-minute periods, obtained by selecting \"sub-hourly\" aggregation period. Other aggregation periods (\"hourly\", \"daily\") are also available, in which case the results are based on further averages of the 5-minute data. A quality flag is also available for this variable.", + "quality_flag": "rh_flag" + }, + "soil_moisture_5": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_5cm_from_earth_surface", + "long_name": "soil_moisture_5cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 5 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_temperature_5": { + "units": "K", + "name_for_output": "soil_temperature_5cm_from_earth_surface", + "long_name": "soil_temperature_5cm_from_earth_surface", + "description": "Average soil temperature measured at 5 cm below the surface level over the aggregation period." + }, + "wetness": { + "units": "Ohms", + "name_for_output": "wetness", + "long_name": "wetness", + "description": "This parameter indicates the presence or absence of moisture due to precipitation, in Ohms. High values (>= 1000) indicate an absence of moisture. Low values (< 1000) indicate the presence of moisture.", + "quality_flag": "wet_flag" + }, + "wind_1_5": { + "units": "m s-1", + "name_for_output": "2m_wind_speed", + "long_name": "2m_wind_speed", + "description": "This parameter is the horizontal velocity of the air near the surface. It is measured using a 3-cup anemometer placed at the same height at the air temperature shield intake. The exact measurement height is 1.5 metres above the ground surface.", + "quality_flag": "wind_flag" + }, + "sr_flag": { + "name_for_output": "solar_irradiance_quality_flag", + "long_name": "solar_irradiance_quality_flag", + "description": "Quality indicator variable 'solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + }, + "st_flag": { + "name_for_output": "soil_temperature_processing_level_quality_flag", + "long_name": "soil_temperature_processing_level_quality_flag", + "description": "A value of 0 in this parameter indicates that the data processing of soil temperature did not return any error." + }, + "rh_flag": { + "name_for_output": "relative_humidity_quality_flag", + "long_name": "relative_humidity_quality_flag", + "description": "This parameter indicates if the \"Relative humidity\" (described in this list) may be used because it is based on an average of good data (value of 0), or if it should be treated with suspicion because erroneous data were detected (value of 3)." + }, + "wet_flag": { + "name_for_output": "wetness_quality_flag", + "long_name": "wetness_quality_flag", + "description": "This parameter indicates if the \"Wetness\" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (non-zero value)." + }, + "wind_flag": { + "name_for_output": "2m_wind_speed_quality_flag", + "long_name": "2m_wind_speed_quality_flag", + "description": "This parameter indicates if the \"2m wind speed\" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (non-zero value)." + }, + "st_type": { + "name_for_output": "soil_temperature_processing_level", + "long_name": "soil_temperature_processing_level", + "description": "This parameter indicates the level of processing applied to the soil temperature measurement: raw data ('R'), corrected data ('C'). A letter 'U' indicates that this information is unknown." + } } - } + }, + "uscrn_hourly": { + "data_table": "unc_hourly", + "order_by": [ + "station_name", + "report_timestamp" + ], + "mandatory_columns": [ + "sitename", + "wbanno", + "longitude", + "latitude", + "date_of_observation" + ], + "products": [ + { + "group_name": "variables", + "columns": [ + "t", + "t_max", + "t_min", + "p_calc", + "solarad", + "solarad_max", + "solarad_min", + "sur_temp", + "sur_temp_max", + "sur_temp_min", + "rh_hr_avg", + "soil_moisture_5", + "soil_moisture_10", + "soil_moisture_20", + "soil_moisture_50", + "soil_moisture_100", + "soil_temp_5", + "soil_temp_10", + "soil_temp_20", + "soil_temp_50", + "soil_temp_100" + ] + }, + { + "group_name": "random_uncertainty", + "columns": [ + "random_pm" + ] + }, + { + "group_name": "positive_total_uncertainty", + "columns": [ + "terr_p" + ] + }, + { + "group_name": "negative_total_uncertainty", + "columns": [ + "terr_m" + ] + }, + { + "group_name": "max_positive_total_uncertainty", + "columns": [ + "tmaxerr_p" + ] + }, + { + "group_name": "max_negative_total_uncertainty", + "columns": [ + "tmaxerr_m" + ] + }, + { + "group_name": "min_positive_total_uncertainty", + "columns": [ + "tminerr_p" + ] + }, + { + "group_name": "min_negative_total_uncertainty", + "columns": [ + "tminerr_m" + ] + }, + { + "group_name": "positive_systematic_uncertainty", + "columns": [ + "sys_p" + ] + }, + { + "group_name": "negative_systematic_uncertainty", + "columns": [ + "sys_m" + ] + }, + { + "group_name": "positive_quasisystematic_uncertainty", + "columns": [ + "quasisys_p" + ] + }, + { + "group_name": "negative_quasisystematic_uncertainty", + "columns": [ + "quasisys_m" + ] + }, + { + "group_name": "quality_flag", + "columns": [ + "solarad_flag", + "solarad_max_flag", + "solarad_min_flag", + "sur_temp_flag", + "sur_temp_max_flag", + "sur_temp_min_flag", + "rh_hr_avg_flag" + ] + }, + { + "group_name": "processing_level", + "columns": [ + "sur_temp_type" + ] + } + ], + "descriptions": { + "id": { + "name_for_output": "report_id", + "description": "Link to header information." + }, + "sitename": { + "name_for_output": "alternative_name", + "description": "This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station \"VA_Charlottesville_2_SSE\" refers to a station located 2 miles South-South-East of Charlottesville, Virginia." + }, + "wbanno": { + "name_for_output": "station_name", + "description": "This is the station identification code." + }, + "longitude": { + "units": "degree_east", + "name_for_output": "longitude", + "long_name": "longitude", + "description": "Longitude of the measurement station, -180.0 to 180.0." + }, + "latitude": { + "units": "degree_north", + "name_for_output": "latitude", + "long_name": "latitude", + "description": "Latitude of the measurement station, -90 to 90." + }, + "t": { + "units": "K", + "name_for_output": "air_temperature", + "long_name": "average_air_temperature", + "description": "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties.", + "positive_total_uncertainty": "terr_p", + "negative_total_uncertainty": "terr_m", + "random_uncertainty": "random_pm", + "positive_systematic_uncertainty": "sys_p", + "negative_systematic_uncertainty": "sys_m", + "positive_quasisystematic_uncertainty": "quasisys_p", + "negative_quasisystematic_uncertainty": "quasisys_m" + }, + "terr_p": { + "units": "K", + "name_for_output": "air_temperature_positive_total_uncertainty", + "long_name": "air_temperature_positive_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "terr_m": { + "units": "K", + "name_for_output": "air_temperature_negative_total_uncertainty", + "long_name": "air_temperature_negative_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "random_pm": { + "units": "K", + "name_for_output": "air_temperature_random_uncertainty", + "long_name": "air_temperature_random_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_p": { + "units": "K", + "name_for_output": "air_temperature_positive_systematic_uncertainty", + "long_name": "air_temperature_positive_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_m": { + "units": "K", + "name_for_output": "air_temperature_negative_systematic_uncertainty", + "long_name": "air_temperature_negative_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmaxerr_p": { + "units": "K", + "name_for_output": "maximum_air_temperature_positive_total_uncertainty", + "long_name": "maximum_air_temperature_positive_total_uncertainty", + "description": "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmaxerr_m": { + "units": "K", + "name_for_output": "maximum_air_temperature_negative_total_uncertainty", + "long_name": "maximum_air_temperature_negative_total_uncertainty", + "description": "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tminerr_p": { + "units": "K", + "name_for_output": "minimum_air_temperature_positive_total_uncertainty", + "long_name": "minimum_air_temperature_positive_total_uncertainty", + "description": "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tminerr_m": { + "units": "K", + "name_for_output": "minimum_air_temperature_negative_total_uncertainty", + "long_name": "minimum_air_temperature_negative_total_uncertainty", + "description": "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "quasisys_p": { + "units": "K", + "name_for_output": "air_temperature_positive_quasisystematic_uncertainty", + "long_name": "air_temperature_positive_quasisystematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "quasisys_m": { + "units": "K", + "name_for_output": "air_temperature_negative_quasisystematic_uncertainty", + "long_name": "air_temperature_negative_quasisystematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "date_of_observation": { + "name_for_output": "report_timestamp", + "description": "This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0)." + }, + "crx_vn": { + "name_for_output": "logbook_version", + "description": "The version number of the station datalogger program that was in effect at the time of the observation." + }, + "t_max": { + "units": "K", + "name_for_output": "maximum_air_temperature", + "long_name": "maximum_air_temperature", + "description": "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties.", + "max_positive_total_uncertainty": "tmaxerr_p", + "max_negative_total_uncertainty": "tmaxerr_m" + }, + "t_min": { + "units": "K", + "name_for_output": "minimum_air_temperature", + "long_name": "minimum_air_temperature", + "description": "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties.", + "min_positive_total_uncertainty": "tminerr_p", + "min_negative_total_uncertainty": "tminerr_m" + }, + "p_calc": { + "units": "mm", + "name_for_output": "accumulated_precipitation", + "long_name": "accumulated_precipitation", + "description": "This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area." + }, + "solarad": { + "units": "W m-2", + "name_for_output": "solar_irradiance", + "long_name": "solar_irradiance", + "description": "The quantity with standard name solar_irradiance, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. \"Irradiance\" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy.", + "quality_flag": "solarad_flag" + }, + "solarad_flag": { + "name_for_output": "solar_irradiance_quality_flag", + "long_name": "solar_irradiance_quality_flag", + "description": "Quality indicator variable 'solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + }, + "solarad_max": { + "units": "W m-2", + "name_for_output": "maximum_solar_irradiance", + "long_name": "maximum_solar_irradiance", + "description": "Maximum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI units at the Earth surface over specified period.", + "quality_flag": "solarad_max_flag" + }, + "solarad_max_flag": { + "name_for_output": "maximum_solar_irradiance_quality_flag", + "long_name": "maximum_solar_irradiance_quality_flag", + "description": "Quality indicator for the variable 'maximum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + }, + "solarad_min": { + "units": "W m-2", + "name_for_output": "minimum_solar_irradiance", + "long_name": "minimum_downward_shortwave_irradiance_at_earth_surfac", + "description": "Minimum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI unitsat the Earth surface.", + "quality_flag": "solarad_min_flag" + }, + "solarad_min_flag": { + "name_for_output": "minimum_solar_irradiance_quality_flag", + "long_name": "minimum_solar_irradiance_quality_flag", + "description": "Quality indicator for variable 'minimum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + }, + "sur_temp": { + "units": "K", + "name_for_output": "soil_temperature", + "long_name": "soil_temperature", + "description": "Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period.", + "quality_flag": "sur_temp_flag", + "processing_level": "sur_temp_type" + }, + "sur_temp_flag": { + "name_for_output": "soil_temperature_quality_flag", + "long_name": "soil_temperature_quality_flag", + "description": "This parameter indicates if the \"Soil temperature\" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3)." + }, + "sur_temp_type": { + "name_for_output": "soil_temperature_processing_level", + "long_name": "soil_temperature_processing_level", + "description": "This parameter indicates the level of processing applied to the soil temperature measurement: raw data ('R'), corrected data ('C'). A letter 'U' indicates that this information is unknown." + }, + "sur_temp_max": { + "units": "K", + "name_for_output": "maximum_soil_temperature", + "long_name": "maximum_soil_temperature", + "description": "From measurements of \"soil temperature\" (described in this list), this parameter provides the highest value measured during the aggregation period.", + "quality_flag": "sur_temp_max_flag" + }, + "sur_temp_max_flag": { + "name_for_output": "maximum_soil_temperature_quality_flag", + "long_name": "maximum_soil_temperature_quality_flag", + "description": "This parameter indicates if the \"Maximum soil temperature\" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). denotes good data and 3 denotes erroneous data." + }, + "sur_temp_min": { + "units": "K", + "name_for_output": "minimum_soil_temperature", + "long_name": "minimum_soil_temperature", + "description": "From measurements of \"soil temperature\" (described in this list), this parameter provides the lowest value measured during the aggregation period.", + "quality_flag": "sur_temp_min_flag" + }, + "sur_temp_min_flag": { + "name_for_output": "minimum_soil_temperature_quality_flag", + "long_name": "minimum_soil_temperature_quality_flag", + "description": "This parameter indicates if the \"Minimum soil temperature\" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3)." + }, + "rh_hr_avg": { + "units": "%", + "name_for_output": "relative_humidity", + "long_name": "relative_humidity", + "description": "This parameter is the ratio of the amount of atmospheric moisture present in the air relative to the amount that would be present if the air were saturated with respect to water or ice. This ratio is multiplied by 100 to provide numbers in percent (%). The measurements are taken at a height of 2 metres above the ground surface. The instrument is a capacitive thin-film device. It measures the flow of electricity across two electrodes separated by a polymer film. The capacitance of the film is converted to relative humidity by a calibration equation. The values are averaged over 5-minute periods, obtained by selecting \"sub-hourly\" aggregation period. Other aggregation periods (\"hourly\", \"daily\") are also available, in which case the results are based on further averages of the 5-minute data. A quality flag is also available for this variable.", + "quality_flag": "rh_hr_avg_flag" + }, + "rh_hr_avg_flag": { + "name_for_output": "relative_humidity_quality_flag", + "long_name": "relative_humidity_quality_flag", + "description": "This parameter indicates if the \"Relative humidity\" (described in this list) may be used because it is based on an average of good data (value of 0), or if it should be treated with suspicion because erroneous data were detected (value of 3)." + }, + "soil_moisture_5": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_5cm_from_earth_surface", + "long_name": "soil_moisture_5cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 5 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_10": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_10cm_from_earth_surface", + "long_name": "soil_moisture_10cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 10 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_20": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_20cm_from_earth_surface", + "long_name": "soil_moisture_20cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 20 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_50": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_50cm_from_earth_surface", + "long_name": "soil_moisture_50cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 50 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_100": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_100cm_from_earth_surface", + "long_name": "soil_moisture_100cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 100 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_temp_5": { + "units": "K", + "name_for_output": "soil_temperature_5cm_from_earth_surface", + "long_name": "soil_temperature_5cm_from_earth_surface", + "description": "Average soil temperature measured at 5 cm below the surface level over the aggregation period." + }, + "soil_temp_10": { + "units": "K", + "name_for_output": "soil_temperature_10cm_from_earth_surface", + "long_name": "soil_temperature_10cm_from_earth_surface", + "description": "Average soil temperature measured at 10 cm below the surface level over the aggregation period." + }, + "soil_temp_20": { + "units": "K", + "name_for_output": "soil_temperature_20cm_from_earth_surface", + "long_name": "soil_temperature_20cm_from_earth_surface", + "description": "Average soil temperature measured at 20 cm below the surface level over the aggregation period." + }, + "soil_temp_50": { + "units": "K", + "name_for_output": "soil_temperature_50cm_from_earth_surface", + "long_name": "soil_temperature_50cm_from_earth_surface", + "description": "Average soil temperature measured at 50 cm below the surface level over the aggregation period." + }, + "soil_temp_100": { + "units": "K", + "name_for_output": "soil_temperature_100cm_from_earth_surface", + "long_name": "soil_temperature_100cm_from_earth_surface", + "description": "Average soil temperature measured at 100 cm below the surface level over the aggregation period." + } + } + }, + "uscrn_daily": { + "data_table": "unc_daily", + "order_by": [ + "station_name", + "report_timestamp" + ], + "mandatory_columns": [ + "sitename", + "wbanno", + "longitude", + "latitude", + "date_of_observation" + ], + "products": [ + { + "group_name": "variables", + "columns": [ + "t", + "t_daily_max", + "t_daily_min", + "t_daily_mean", + "p_daily_calc", + "solarad_daily", + "sur_temp_daily_max", + "sur_temp_daily_min", + "sur_temp_daily_avg", + "rh_daily_max", + "rh_daily_min", + "rh_daily_avg", + "soil_moisture_5_daily", + "soil_moisture_10_daily", + "soil_moisture_20_daily", + "soil_moisture_50_daily", + "soil_moisture_100_daily", + "soil_temp_5_daily", + "soil_temp_10_daily", + "soil_temp_20_daily", + "soil_temp_50_daily", + "soil_temp_100_daily" + ] + }, + { + "group_name": "positive_random_uncertainty", + "columns": [ + "random_p" + ] + }, + { + "group_name": "negative_random_uncertainty", + "columns": [ + "random_m" + ] + }, + { + "group_name": "positive_total_uncertainty", + "columns": [ + "terr_p", + "tmaxerr_p", + "tminerr_p", + "tmeanerr_p" + ] + }, + { + "group_name": "negative_total_uncertainty", + "columns": [ + "terr_m", + "tmaxerr_m", + "tminerr_m", + "tmeanerr_m" + ] + }, + { + "group_name": "positive_quasisystematic_uncertainty", + "columns": [ + "quasisys_p" + ] + }, + { + "group_name": "negative_quasisystematic_uncertainty", + "columns": [ + "quasisys_m" + ] + }, + { + "group_name": "positive_systematic_uncertainty", + "columns": [ + "sys_p" + ] + }, + { + "group_name": "negative_systematic_uncertainty", + "columns": [ + "sys_m" + ] + }, + { + "group_name": "processing_level", + "columns": [ + "sur_temp_daily_type" + ] + } + ], + "descriptions": { + "id": { + "name_for_output": "report_id", + "description": "Link to header information." + }, + "sitename": { + "name_for_output": "alternative_name", + "description": "This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station \"VA_Charlottesville_2_SSE\" refers to a station located 2 miles South-South-East of Charlottesville, Virginia." + }, + "wbanno": { + "name_for_output": "station_name", + "description": "This is the station identification code." + }, + "longitude": { + "units": "degree_east", + "name_for_output": "longitude", + "long_name": "longitude", + "description": "Longitude of the measurement station, -180.0 to 180.0." + }, + "latitude": { + "units": "degree_north", + "name_for_output": "latitude", + "long_name": "latitude", + "description": "Latitude of the measurement station, -90 to 90." + }, + "t": { + "units": "K", + "name_for_output": "air_temperature", + "long_name": "average_air_temperature", + "description": "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties.", + "positive_systematic_uncertainty": "sys_p", + "negative_systematic_uncertainty": "sys_m", + "positive_total_uncertainty": "terr_p", + "negative_total_uncertainty": "terr_m", + "positive_random_uncertainty": "random_p", + "negative_random_uncertainty": "random_m" + }, + "terr_p": { + "units": "K", + "name_for_output": "air_temperature_positive_total_uncertainty", + "long_name": "air_temperature_positive_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "terr_m": { + "units": "K", + "name_for_output": "air_temperature_negative_total_uncertainty", + "long_name": "air_temperature_negative_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmaxerr_p": { + "units": "K", + "name_for_output": "maximum_air_temperature_positive_total_uncertainty", + "long_name": "maximum_air_temperature_positive_total_uncertainty", + "description": "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmaxerr_m": { + "units": "K", + "name_for_output": "maximum_air_temperature_negative_total_uncertainty", + "long_name": "maximum_air_temperature_negative_total_uncertainty", + "description": "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tminerr_p": { + "units": "K", + "name_for_output": "minimum_air_temperature_positive_total_uncertainty", + "long_name": "minimum_air_temperature_positive_total_uncertainty", + "description": "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tminerr_m": { + "units": "K", + "name_for_output": "minimum_air_temperature_negative_total_uncertainty", + "long_name": "minimum_air_temperature_negative_total_uncertainty", + "description": "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmeanerr_p": { + "units": "K", + "name_for_output": "mean_air_temperature_positive_total_uncertainty", + "long_name": "mean_air_temperature_positive_total_uncertainty", + "description": "For \"Mean air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmeanerr_m": { + "units": "K", + "name_for_output": "mean_air_temperature_negative_total_uncertainty", + "long_name": "mean_air_temperature_negative_total_uncertainty", + "description": "For \"Mean air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "random_p": { + "units": "K", + "name_for_output": "air_temperature_positive_random_uncertainty", + "long_name": "air_temperature_positive_random_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "random_m": { + "units": "K", + "name_for_output": "air_temperature_negative_random_uncertainty", + "long_name": "air_temperature_negative_random_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_p": { + "units": "K", + "name_for_output": "air_temperature_positive_systematic_uncertainty", + "long_name": "air_temperature_positive_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_m": { + "units": "K", + "name_for_output": "air_temperature_negative_systematic_uncertainty", + "long_name": "air_temperature_negative_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "quasisys_p": { + "units": "K", + "name_for_output": "air_temperature_positive_quasisystematic_uncertainty", + "long_name": "air_temperature_positive_quasisystematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "quasisys_m": { + "units": "K", + "name_for_output": "air_temperature_negative_quasisystematic_uncertainty", + "long_name": "air_temperature_negative_quasisystematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "date_of_observation": { + "name_for_output": "report_timestamp", + "description": "This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0)." + }, + "crx_vn": { + "name_for_output": "logbook_version", + "description": "LogBook software and version." + }, + "t_daily_max": { + "units": "K", + "name_for_output": "maximum_air_temperature", + "long_name": "maximum_air_temperature", + "description": "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties.", + "positive_total_uncertainty": "tmaxerr_p", + "negative_total_uncertainty": "tmaxerr_m" + }, + "t_daily_min": { + "units": "K", + "name_for_output": "minimum_air_temperature", + "long_name": "minimum_air_temperature", + "description": "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties.", + "positive_total_uncertainty": "tminerr_p", + "negative_total_uncertainty": "tminerr_m" + }, + "t_daily_mean": { + "units": "K", + "name_for_output": "mean_air_temperature", + "long_name": "mean_air_temperature", + "description": "This parameter is the average of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties.", + "positive_total_uncertainty": "tmeanerr_p", + "negative_total_uncertainty": "tmeanerr_m" + }, + "p_daily_calc": { + "units": "mm", + "name_for_output": "accumulated_precipitation", + "long_name": "accumulated_precipitation", + "description": "This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area." + }, + "solarad_daily": { + "units": "MJ m-2", + "name_for_output": "daily_global_solar_radiation", + "long_name": "daily_global_solar_radiation", + "description": "This parameter is the total solar energy(direct and diffuse) measured at the Earth's surface per unit of area. in MJ/meter^2, calculated from the hourly average global solar radiation rates and converted to energy by integrating over time. For simplicity of use, the values are provided in Mega-Joules per square metre (MJ/m**2)." + }, + "sur_temp_daily_type": { + "name_for_output": "soil_temperature_processing_level", + "long_name": "soil_temperature_processing_level", + "description": "This parameter indicates the level of processing applied to the soil temperature measurement: raw data ('R'), corrected data ('C'). A letter 'U' indicates that this information is unknown." + }, + "sur_temp_daily_max": { + "units": "K", + "name_for_output": "maximum_soil_temperature", + "long_name": "maximum_soil_temperature", + "description": "From measurements of \"soil temperature\" (described in this list), this parameter provides the highest value measured during the aggregation period." + }, + "sur_temp_daily_min": { + "units": "K", + "name_for_output": "minimum_soil_temperature", + "long_name": "minimum_soil_temperature", + "description": "From measurements of \"soil temperature\" (described in this list), this parameter provides the lowest value measured during the aggregation period." + }, + "sur_temp_daily_avg": { + "units": "K", + "name_for_output": "soil_temperature", + "long_name": "soil_temperature", + "description": "Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period.", + "processing_level": "sur_temp_daily_type" + }, + "rh_daily_max": { + "units": "%", + "name_for_output": "maximum_relative_humidity", + "long_name": "maximum_relative_humidity", + "description": "From measurements of \"Relative humidity\" (described in this list), this parameter provides the highest value measured during the aggregation period." + }, + "rh_daily_min": { + "units": "%", + "name_for_output": "minimum_relative_humidity", + "long_name": "minimum_relative_humidity", + "description": "From measurements of \"Relative humidity\" (described in this list), this parameter provides the lowest value measured during the aggregation period." + }, + "rh_daily_avg": { + "units": "%", + "name_for_output": "relative_humidity", + "long_name": "relative_humidity", + "description": "This parameter is the ratio of the amount of atmospheric moisture present in the air relative to the amount that would be present if the air were saturated with respect to water or ice. This ratio is multiplied by 100 to provide numbers in percent (%). The measurements are taken at a height of 2 metres above the ground surface. The instrument is a capacitive thin-film device. It measures the flow of electricity across two electrodes separated by a polymer film. The capacitance of the film is converted to relative humidity by a calibration equation. The values are averaged over 5-minute periods, obtained by selecting \"sub-hourly\" aggregation period. Other aggregation periods (\"hourly\", \"daily\") are also available, in which case the results are based on further averages of the 5-minute data. A quality flag is also available for this variable." + }, + "soil_moisture_5_daily": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_5cm_from_earth_surface", + "long_name": "soil_moisture_5cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 5 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_10_daily": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_10cm_from_earth_surface", + "long_name": "soil_moisture_10cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 10 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_20_daily": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_20cm_from_earth_surface", + "long_name": "soil_moisture_20cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 20 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_50_daily": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_50cm_from_earth_surface", + "long_name": "soil_moisture_50cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 50 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_moisture_100_daily": { + "units": "m3 m-3", + "name_for_output": "soil_moisture_100cm_from_earth_surface", + "long_name": "soil_moisture_100cm_from_earth_surface", + "description": "Average content of liquid water in a surface soil layer of 0 to 100 cm depth expressed as m3 water per m3 soil aggregation period." + }, + "soil_temp_5_daily": { + "units": "K", + "name_for_output": "soil_temperature_5cm_from_earth_surface", + "long_name": "soil_temperature_5cm_from_earth_surface", + "description": "Average soil temperature measured at 5 cm below the surface level over the aggregation period." + }, + "soil_temp_10_daily": { + "units": "K", + "name_for_output": "soil_temperature_10cm_from_earth_surface", + "long_name": "soil_temperature_10cm_from_earth_surface", + "description": "Average soil temperature measured at 10 cm below the surface level over the aggregation period." + }, + "soil_temp_20_daily": { + "units": "K", + "name_for_output": "soil_temperature_20cm_from_earth_surface", + "long_name": "soil_temperature_20cm_from_earth_surface", + "description": "Average soil temperature measured at 20 cm below the surface level over the aggregation period." + }, + "soil_temp_50_daily": { + "units": "K", + "name_for_output": "soil_temperature_50cm_from_earth_surface", + "long_name": "soil_temperature_50cm_from_earth_surface", + "description": "Average soil temperature measured at 50 cm below the surface level over the aggregation period." + }, + "soil_temp_100_daily": { + "units": "K", + "name_for_output": "soil_temperature_100cm_from_earth_surface", + "long_name": "soil_temperature_100cm_from_earth_surface", + "description": "Average soil temperature measured at 100 cm below the surface level over the aggregation period." + } + } + }, + "uscrn_monthly": { + "data_table": "unc_monthly", + "order_by": [ + "station_name", + "report_timestamp" + ], + "mandatory_columns": [ + "sitename", + "wbanno", + "longitude", + "latitude", + "date_of_observation" + ], + "products": [ + { + "group_name": "variables", + "columns": [ + "t", + "t_monthly_max", + "t_monthly_min", + "t_monthly_mean", + "p_monthly_calc", + "solrad_monthly_avg", + "sur_temp_monthly_max", + "sur_temp_monthly_min", + "sur_temp_monthly_avg" + ] + }, + { + "group_name": "positive_random_uncertainty", + "columns": [ + "random_p" + ] + }, + { + "group_name": "negative_random_uncertainty", + "columns": [ + "random_m" + ] + }, + { + "group_name": "positive_total_uncertainty", + "columns": [ + "terr_p", + "tmaxerr_p", + "tminerr_p" + ] + }, + { + "group_name": "negative_total_uncertainty", + "columns": [ + "terr_m", + "tmaxerr_m", + "tminerr_m" + ] + }, + { + "group_name": "positive_quasisystematic_uncertainty", + "columns": [ + "quasisys_p" + ] + }, + { + "group_name": "negative_quasisystematic_uncertainty", + "columns": [ + "quasisys_m" + ] + }, + { + "group_name": "positive_systematic_uncertainty", + "columns": [ + "sys_p" + ] + }, + { + "group_name": "negative_systematic_uncertainty", + "columns": [ + "sys_m" + ] + }, + { + "group_name": "processing_level", + "columns": [ + "sur_temp_monthly_type" + ] + } + ], + "descriptions": { + "id": { + "name_for_output": "report_id", + "description": "Link to header information." + }, + "sitename": { + "name_for_output": "alternative_name", + "description": "This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station \"VA_Charlottesville_2_SSE\" refers to a station located 2 miles South-South-East of Charlottesville, Virginia." + }, + "wbanno": { + "name_for_output": "station_name", + "description": "This is the station identification code." + }, + "t": { + "units": "K", + "name_for_output": "air_temperature", + "long_name": "average_air_temperature", + "description": "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties.", + "positive_systematic_uncertainty": "sys_p", + "negative_systematic_uncertainty": "sys_m", + "positive_total_uncertainty": "terr_p", + "negative_total_uncertainty": "terr_m", + "positive_random_uncertainty": "random_p", + "negative_random_uncertainty": "random_m", + "positive_quasisystematic_uncertainty": "quasisys_p", + "negative_quasisystematic_uncertainty": "quasisys_m" + }, + "terr_p": { + "units": "K", + "name_for_output": "air_temperature_positive_total_uncertainty", + "long_name": "air_temperature_positive_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "terr_m": { + "units": "K", + "name_for_output": "air_temperature_negative_total_uncertainty", + "long_name": "air_temperature_negative_total_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmaxerr_p": { + "units": "K", + "name_for_output": "maximum_air_temperature_positive_total_uncertainty", + "long_name": "maximum_air_temperature_positive_total_uncertainty", + "description": "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tmaxerr_m": { + "units": "K", + "name_for_output": "maximum_air_temperature_negative_total_uncertainty", + "long_name": "maximum_air_temperature_negative_total_uncertainty", + "description": "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tminerr_p": { + "units": "K", + "name_for_output": "minimum_air_temperature_positive_total_uncertainty", + "long_name": "minimum_air_temperature_positive_total_uncertainty", + "description": "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "tminerr_m": { + "units": "K", + "name_for_output": "minimum_air_temperature_negative_total_uncertainty", + "long_name": "minimum_air_temperature_negative_total_uncertainty", + "description": "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "random_p": { + "units": "K", + "name_for_output": "air_temperature_positive_random_uncertainty", + "long_name": "air_temperature_positive_random_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "random_m": { + "units": "K", + "name_for_output": "air_temperature_negative_random_uncertainty", + "long_name": "air_temperature_negative_random_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_p": { + "units": "K", + "name_for_output": "air_temperature_positive_systematic_uncertainty", + "long_name": "air_temperature_positive_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "sys_m": { + "units": "K", + "name_for_output": "air_temperature_negative_systematic_uncertainty", + "long_name": "air_temperature_negative_systematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "quasisys_p": { + "units": "K", + "name_for_output": "air_temperature_positive_quasisystematic_uncertainty", + "long_name": "air_temperature_positive_quasisystematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "quasisys_m": { + "units": "K", + "name_for_output": "air_temperature_negative_quasisystematic_uncertainty", + "long_name": "air_temperature_negative_quasisystematic_uncertainty", + "description": "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + }, + "date_of_observation": { + "name_for_output": "report_timestamp", + "description": "This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0)." + }, + "crx_vn_monthly": { + "name_for_output": "logbook_version", + "description": "LogBook software and version." + }, + "longitude": { + "units": "degree_east", + "name_for_output": "longitude", + "long_name": "longitude", + "description": "Longitude of the measurement station, -180.0 to 180.0." + }, + "latitude": { + "units": "degree_north", + "name_for_output": "latitude", + "long_name": "latitude", + "description": "Latitude of the measurement station, -90 to 90." + }, + "t_monthly_max": { + "units": "K", + "name_for_output": "maximum_air_temperature", + "long_name": "maximum_air_temperature", + "description": "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties.", + "positive_total_uncertainty": "tmaxerr_p", + "negative_total_uncertainty": "tmaxerr_m" + }, + "t_monthly_min": { + "units": "K", + "name_for_output": "minimum_air_temperature", + "long_name": "minimum_air_temperature", + "description": "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties.", + "positive_total_uncertainty": "tminerr_p", + "negative_total_uncertainty": "tminerr_m" + }, + "t_monthly_mean": { + "units": "K", + "name_for_output": "mean_air_temperature", + "long_name": "mean_air_temperature", + "description": "This parameter is the average of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." + }, + "p_monthly_calc": { + "units": "mm", + "name_for_output": "accumulated_precipitation", + "long_name": "accumulated_precipitation", + "description": "This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area." + }, + "solrad_monthly_avg": { + "units": "MJ m-2", + "name_for_output": "monthly_global_solar_radiation", + "long_name": "monthly_global_solar_radiation", + "description": "This parameter is the total solar energy(direct and diffuse) measured at the Earth's surface per unit of area. in MJ/meter^2, calculated from the daily average global solar radiation rates and converted to energy by integrating over time. For simplicity of use, the values are provided in Mega-Joules per square metre (MJ/m**2)." + }, + "sur_temp_monthly_type": { + "name_for_output": "soil_temperature_processing_level", + "long_name": "soil_temperature_processing_level", + "description": "This parameter indicates the level of processing applied to the soil temperature measurement: raw data ('R'), corrected data ('C'). A letter 'U' indicates that this information is unknown." + }, + "sur_temp_monthly_max": { + "units": "K", + "name_for_output": "maximum_soil_temperature", + "long_name": "maximum_soil_temperature", + "description": "From measurements of \"soil temperature\" (described in this list), this parameter provides the highest value measured during the aggregation period." + }, + "sur_temp_monthly_min": { + "units": "K", + "name_for_output": "minimum_soil_temperature", + "long_name": "minimum_soil_temperature", + "description": "From measurements of \"soil temperature\" (described in this list), this parameter provides the lowest value measured during the aggregation period." + }, + "sur_temp_monthly_avg": { + "units": "K", + "name_for_output": "soil_temperature", + "long_name": "soil_temperature", + "description": "Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period.", + "processing_level": "sur_temp_monthly_type" + } + } + } + } } diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml index 757bb3f..3f69312 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml @@ -40,12 +40,12 @@ out_columns_order: - soil_temperature_max_quality_flag - soil_temperature_min - soil_temperature_min_quality_flag -- downward_shortwave_irradiance_at_earth_surface -- downward_shortwave_irradiance_at_earth_surface_quality_flag -- downward_shortwave_irradiance_at_earth_surface_max -- downward_shortwave_irradiance_at_earth_surface_max_quality_flag -- downward_shortwave_irradiance_at_earth_surface_min -- downward_shortwave_irradiance_at_earth_surface_min_quality_flag +- solar_irradiance +- solar_irradiance_quality_flag +- maximum_solar_irradiance +- maximum_solar_irradiance_quality_flag +- minimum_solar_irradiance +- minimum_solar_irradiance_quality_flag - soil_moisture_5cm_from_earth_surface - soil_moisture_10cm_from_earth_surface - soil_moisture_20cm_from_earth_surface @@ -86,7 +86,7 @@ products_hierarchy: - positive_quasisystematic_uncertainty - negative_quasisystematic_uncertainty sources: - USCRN_DAILY: + uscrn_daily: cdm_mapping: melt_columns: true rename: @@ -114,12 +114,16 @@ sources: soil_temp_20_daily: soil_temperature_20cm_from_earth_surface soil_temp_50_daily: soil_temperature_50cm_from_earth_surface soil_temp_5_daily: soil_temperature_5cm_from_earth_surface + solarad_daily: daily_global_solar_radiation sur_temp_daily_avg: soil_temperature + sur_temp_daily_max: maximum_soil_temperature + sur_temp_daily_min: minimum_soil_temperature sur_temp_daily_type: soil_temperature_processing_level sys_m: air_temperature_negative_systematic_uncertainty sys_p: air_temperature_positive_systematic_uncertainty + t: air_temperature t_daily_max: daily_maximum_air_temperature - t_daily_mean: air_temperature + t_daily_mean: daily_mean_air_temperature t_daily_min: daily_minimum_air_temperature terr_m: air_temperature_negative_total_uncertainty terr_p: air_temperature_positive_total_uncertainty @@ -136,206 +140,338 @@ sources: mm: mm offset: 0 scale: 1 + air_temperature: + names: + K: K + offset: 0 + scale: 1 + daily_global_solar_radiation: + names: + MJ m-2: MJ m-2 + offset: 0 + scale: 1 + maximum_air_temperature: + names: + K: K + offset: 0 + scale: 1 + maximum_relative_humidity: + names: + '%': '%' + offset: 0 + scale: 1 + maximum_soil_temperature: + names: + K: K + offset: 0 + scale: 1 + mean_air_temperature: + names: + K: K + offset: 0 + scale: 1 + minimum_air_temperature: + names: + K: K + offset: 0 + scale: 1 + minimum_relative_humidity: + names: + '%': '%' + offset: 0 + scale: 1 + minimum_soil_temperature: + names: + K: K + offset: 0 + scale: 1 + relative_humidity: + names: + '%': '%' + offset: 0 + scale: 1 + soil_moisture_100cm_from_earth_surface: + names: + m3 m-3: m3 m-3 + offset: 0 + scale: 1 + soil_moisture_10cm_from_earth_surface: + names: + m3 m-3: m3 m-3 + offset: 0 + scale: 1 + soil_moisture_20cm_from_earth_surface: + names: + m3 m-3: m3 m-3 + offset: 0 + scale: 1 + soil_moisture_50cm_from_earth_surface: + names: + m3 m-3: m3 m-3 + offset: 0 + scale: 1 + soil_moisture_5cm_from_earth_surface: + names: + m3 m-3: m3 m-3 + offset: 0 + scale: 1 + soil_temperature: + names: + K: K + offset: 0 + scale: 1 + soil_temperature_100cm_from_earth_surface: + names: + K: K + offset: 0 + scale: 1 + soil_temperature_10cm_from_earth_surface: + names: + K: K + offset: 0 + scale: 1 + soil_temperature_20cm_from_earth_surface: + names: + K: K + offset: 0 + scale: 1 + soil_temperature_50cm_from_earth_surface: + names: + K: K + offset: 0 + scale: 1 + soil_temperature_5cm_from_earth_surface: + names: + K: K + offset: 0 + scale: 1 data_table: unc_daily descriptions: accumulated_precipitation: - description: Total amount of precipitation + description: This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area. dtype: float32 - long_name: accumulated_precipitation + long_name: p_daily_calc units: mm air_temperature: - description: 'Mean air temperature calculated using the typical historical approach: (T_DAILY_MAX + T_DAILY_MIN) / 2' + description: "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: mean_air_temperature - negative_total_uncertainty: tmeanerr_m - positive_total_uncertainty: tmeanerr_p - units: K - air_temperature_max_negative_total_uncertainty: - description: negative uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_negative_total_uncertainty - units: K - air_temperature_max_positive_total_uncertainty: - description: positive uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_positive_total_uncertainty - units: K - air_temperature_mean_negative_total_uncertainty: - description: tmeanerr_m, tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (negative per using the Hourly data) - dtype: float32 - long_name: mean_air_temperature_negative_total_uncertainty - units: K - air_temperature_mean_positive_total_uncertainty: - description: tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (positive part using the Hourly data) - dtype: float32 - long_name: mean_air_temperature_positive_total_uncertainty - units: K - air_temperature_min_negative_total_uncertainty: - description: negative uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_negative_total_uncertainty - units: K - air_temperature_min_positive_total_uncertainty: - description: positive uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_positive_total_uncertainty + long_name: t + negative_random_uncertainty: random_m + negative_systematic_uncertainty: sys_m + negative_total_uncertainty: terr_m + positive_random_uncertainty: random_p + positive_systematic_uncertainty: sys_p + positive_total_uncertainty: terr_p units: K air_temperature_negative_quasisystematic_uncertainty: - description: -ve/-ve Quasi-systematic + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_quasisystematic_uncertainty + long_name: quasisys_m units: K air_temperature_negative_random_uncertainty: - description: -ve Random uncertainty + description: "For \"Air temperature\" (described in this list), this parameter one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_random_uncertainty + long_name: random_m units: K air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty + long_name: sys_m units: K air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_total_uncertainty + long_name: terr_m units: K air_temperature_positive_quasisystematic_uncertainty: - description: +ve Quasi-systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_quasisystematic_uncertainty + long_name: quasisys_p units: K air_temperature_positive_random_uncertainty: - description: +ve Random uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_random_uncertainty + long_name: random_p units: K air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty + long_name: sys_p units: K air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_total_uncertainty + long_name: terr_p units: K alternative_name: - description: Alternative name for station + description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object long_name: sitename - daily_maximum_air_temperature: - description: Maximum air temperature + daily_global_solar_radiation: + description: This parameter is the total solar energy(direct and diffuse) measured at the Earth's surface per unit of area. in MJ/meter^2, calculated from the hourly average global solar radiation rates and converted to energy by integrating over time. For simplicity of use, the values are provided in Mega-Joules per square metre (MJ/m**2). + dtype: float32 + long_name: solarad_daily + units: MJ m-2 + latitude: + description: Latitude of the measurement station, -90 to 90. + dtype: float32 + long_name: latitude + units: degree_north + logbook_version: + description: LogBook software and version. + dtype: float32 + long_name: crx_vn + longitude: + description: Longitude of the measurement station, -180.0 to 180.0. + dtype: float32 + long_name: longitude + units: degree_east + maximum_air_temperature: + description: "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: air_temperature_max + long_name: t_daily_max negative_total_uncertainty: tmaxerr_m positive_total_uncertainty: tmaxerr_p units: K - daily_minimum_air_temperature: - description: Minimum air temperature + maximum_air_temperature_negative_total_uncertainty: + description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + dtype: float32 + long_name: tmaxerr_m + units: K + maximum_air_temperature_positive_total_uncertainty: + description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + dtype: float32 + long_name: tmaxerr_p + units: K + maximum_relative_humidity: + description: From measurements of "Relative humidity" (described in this list), this parameter provides the highest value measured during the aggregation period. dtype: float32 - long_name: minimum_air_temperature + long_name: rh_daily_max + units: '%' + maximum_soil_temperature: + description: From measurements of "soil temperature" (described in this list), this parameter provides the highest value measured during the aggregation period. + dtype: float32 + long_name: sur_temp_daily_max + units: K + mean_air_temperature: + description: "This parameter is the average of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." + dtype: float32 + long_name: t_daily_mean + negative_total_uncertainty: tmeanerr_m + positive_total_uncertainty: tmeanerr_p + units: K + mean_air_temperature_negative_total_uncertainty: + description: "For \"Mean air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + dtype: float32 + long_name: tmeanerr_m + units: K + mean_air_temperature_positive_total_uncertainty: + description: "For \"Mean air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + dtype: float32 + long_name: tmeanerr_p + units: K + minimum_air_temperature: + description: "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." + dtype: float32 + long_name: t_daily_min negative_total_uncertainty: tminerr_m positive_total_uncertainty: tminerr_p units: K - location_latitude: - description: Latitude of the station (deg. North) + minimum_air_temperature_negative_total_uncertainty: + description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: latitude - units: degree_north - location_longitude: - description: Longitude of the station (deg. East) + long_name: tminerr_m + units: K + minimum_air_temperature_positive_total_uncertainty: + description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: longitude - units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn - relative_humidity: - description: Relative humidity is computed from 5-minute values in almost all cases. All USCRN stations now report 5-minute averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. + long_name: tminerr_p + units: K + minimum_relative_humidity: + description: From measurements of "Relative humidity" (described in this list), this parameter provides the lowest value measured during the aggregation period. dtype: float32 - long_name: average_relative_humidity + long_name: rh_daily_min units: '%' - daily_maximum_relative_humidity: - description: Maximum relative humidity + minimum_soil_temperature: + description: From measurements of "soil temperature" (described in this list), this parameter provides the lowest value measured during the aggregation period. dtype: float32 - long_name: maximum_relative_humidity - units: '%' - daily_minimum_relative_humidity: - description: Minimum relative humidity + long_name: sur_temp_daily_min + units: K + relative_humidity: + description: This parameter is the ratio of the amount of atmospheric moisture present in the air relative to the amount that would be present if the air were saturated with respect to water or ice. This ratio is multiplied by 100 to provide numbers in percent (%). The measurements are taken at a height of 2 metres above the ground surface. The instrument is a capacitive thin-film device. It measures the flow of electricity across two electrodes separated by a polymer film. The capacitance of the film is converted to relative humidity by a calibration equation. The values are averaged over 5-minute periods, obtained by selecting "sub-hourly" aggregation period. Other aggregation periods ("hourly", "daily") are also available, in which case the results are based on further averages of the 5-minute data. A quality flag is also available for this variable. dtype: float32 - long_name: minimum_relative_humidity + long_name: rh_daily_avg units: '%' report_id: - description: Identifier in the USCRN meta-database + description: Link to header information. + dtype: float32 long_name: id report_timestamp: - description: observation date time UTC + description: This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0). dtype: datetime64[ns] long_name: date_of_observation soil_moisture_100cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 100 cm below the surface. + description: Average content of liquid water in a surface soil layer of 0 to 100 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_100cm_from_earth_surface + long_name: soil_moisture_100_daily units: m3 m-3 soil_moisture_10cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 10 cm below the surface. + description: Average content of liquid water in a surface soil layer of 0 to 10 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_10cm_from_earth_surface + long_name: soil_moisture_10_daily units: m3 m-3 soil_moisture_20cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 20 cm below the surface. + description: Average content of liquid water in a surface soil layer of 0 to 20 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_20cm_from_earth_surface + long_name: soil_moisture_20_daily units: m3 m-3 soil_moisture_50cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 50 cm below the surface. + description: Average content of liquid water in a surface soil layer of 0 to 50 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_50cm_from_earth_surface + long_name: soil_moisture_50_daily units: m3 m-3 soil_moisture_5cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 5 cm below the surface. + description: Average content of liquid water in a surface soil layer of 0 to 5 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_5cm_from_earth_surface + long_name: soil_moisture_5_daily units: m3 m-3 soil_temperature: - description: Average infrared surface temperature + description: Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period. dtype: float32 - long_name: average_soil_temperature + long_name: sur_temp_daily_avg processing_level: sur_temp_daily_type units: K soil_temperature_100cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 100 cm below the surface. + description: Average soil temperature measured at 100 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_100cm_from_earth_surface + long_name: soil_temp_100_daily units: K soil_temperature_10cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 10 cm below the surface. + description: Average soil temperature measured at 10 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_10cm_from_earth_surface + long_name: soil_temp_10_daily units: K soil_temperature_20cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 20 cm below the surface. + description: Average soil temperature measured at 20 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_20cm_from_earth_surface + long_name: soil_temp_20_daily units: K soil_temperature_50cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 50 cm below the surface. + description: Average soil temperature measured at 50 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_50cm_from_earth_surface + long_name: soil_temp_50_daily units: K soil_temperature_5cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 5 cm below the surface. + description: Average soil temperature measured at 5 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_5cm_from_earth_surface + long_name: soil_temp_5_daily units: K soil_temperature_processing_level: - description: Type of infrared surface temperature measurement. + description: 'This parameter indicates the level of processing applied to the soil temperature measurement: raw data (''R''), corrected data (''C''). A letter ''U'' indicates that this information is unknown.' dtype: float32 - long_name: soil_temperature_processing_level + long_name: sur_temp_daily_type primary_station_id: - description: Station identification code + description: This is the station identification code. dtype: object long_name: wbanno mandatory_columns: @@ -349,10 +485,14 @@ sources: - report_timestamp products: - columns: + - t - t_daily_max - t_daily_min - t_daily_mean - p_daily_calc + - solarad_daily + - sur_temp_daily_max + - sur_temp_daily_min - sur_temp_daily_avg - rh_daily_max - rh_daily_min @@ -401,7 +541,7 @@ sources: - columns: - sur_temp_daily_type group_name: processing_level - USCRN_HOURLY: + uscrn_hourly: cdm_mapping: melt_columns: true rename: @@ -427,12 +567,12 @@ sources: soil_temp_20: soil_temperature_20cm_from_earth_surface soil_temp_5: soil_temperature_5cm_from_earth_surface soil_temp_50: soil_temperature_50cm_from_earth_surface - solarad: downward_shortwave_irradiance_at_earth_surface - solarad_flag: downward_shortwave_irradiance_at_earth_surface_quality_flag - solarad_max: hourly_maximum_downward_shortwave_irradiance_at_earth_surface - solarad_max_flag: hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag - solarad_min: hourly_minimum_downward_shortwave_irradiance_at_earth_surface - solarad_min_flag: hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag + solarad: solar_irradiance + solarad_flag: solar_irradiance_quality_flag + solarad_max: maximum_solar_irradiance + solarad_max_flag: maximum_solar_irradiance_quality_flag + solarad_min: minimum_solar_irradiance + solarad_min_flag: minimum_solar_irradiance_quality_flag sur_temp: soil_temperature sur_temp_flag: soil_temperature_quality_flag sur_temp_max: hourly_maximum_soil_temperature @@ -461,18 +601,14 @@ sources: data_table: unc_hourly descriptions: accumulated_precipitation: - description: Total amount of precipitation, in mm, recorded during the hour + description: This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area. dtype: float32 - long_name: accumulated_precipitation + long_name: p_calc units: mm air_temperature: - description: Near surface temperature + description: "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: average_air_temperature - max_negative_total_uncertainty: tmaxerr_m - max_positive_total_uncertainty: tmaxerr_p - min_negative_total_uncertainty: tminerr_m - min_positive_total_uncertainty: tminerr_p + long_name: t negative_quasisystematic_uncertainty: quasisys_m negative_systematic_uncertainty: sys_m negative_total_uncertainty: terr_m @@ -481,223 +617,228 @@ sources: positive_total_uncertainty: terr_p random_uncertainty: random_pm units: K - air_temperature_max_negative_total_uncertainty: - description: negative uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_negative_total_uncertainty - units: K - air_temperature_max_positive_total_uncertainty: - description: positive uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_positive_total_uncertainty - units: K - air_temperature_min_negative_total_uncertainty: - description: negative uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_negative_total_uncertainty - units: K - air_temperature_min_positive_total_uncertainty: - description: positive uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_positive_total_uncertainty - units: K air_temperature_negative_quasisystematic_uncertainty: - description: -ve/-ve Quasi-systematic + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_quasisystematic_uncertainty + long_name: quasisys_m units: K air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty + long_name: sys_m units: K air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_total_uncertainty + long_name: terr_m units: K air_temperature_positive_quasisystematic_uncertainty: - description: +ve Quasi-systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_quasisystematic_uncertainty + long_name: quasisys_p units: K air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty + long_name: sys_p units: K air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_total_uncertainty + long_name: terr_p units: K air_temperature_random_uncertainty: - description: +ve/-ve Random + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_random_uncertainty + long_name: random_pm units: K alternative_name: - description: Alternative name for station + description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object long_name: sitename - daily_maximum_air_temperature: - description: Maximum air temperature during the hour + latitude: + description: Latitude of the measurement station, -90 to 90. + dtype: float32 + long_name: latitude + units: degree_north + logbook_version: + description: The version number of the station datalogger program that was in effect at the time of the observation. + dtype: float32 + long_name: crx_vn + longitude: + description: Longitude of the measurement station, -180.0 to 180.0. + dtype: float32 + long_name: longitude + units: degree_east + maximum_air_temperature: + description: "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: air_temperature_max + long_name: t_max + max_negative_total_uncertainty: tmaxerr_m + max_positive_total_uncertainty: tmaxerr_p units: K - daily_minimum_air_temperature: - description: Minimum air temperature during the hour + maximum_air_temperature_negative_total_uncertainty: + description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: minimum_air_temperature + long_name: tmaxerr_m units: K - downward_shortwave_irradiance_at_earth_surface: - description: Average global solar radiation + maximum_air_temperature_positive_total_uncertainty: + description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - flag: solarad_flag - long_name: downward_shortwave_irradiance_at_earth_surface - units: W m-2 - hourly_maximum_downward_shortwave_irradiance_at_earth_surface: - description: Maximum global solar radiation + long_name: tmaxerr_p + units: K + maximum_soil_temperature: + description: From measurements of "soil temperature" (described in this list), this parameter provides the highest value measured during the aggregation period. dtype: float32 - flag: solarad_max_flag - long_name: maximum_downward_shortwave_irradiance_at_earth_surface - units: W m-2 - hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for maximum global solar radiation. 0 denotes good data and 3 denotes erroneous data. + long_name: sur_temp_max + quality_flag: sur_temp_max_flag + units: K + maximum_soil_temperature_quality_flag: + description: This parameter indicates if the "Maximum soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). denotes good data and 3 denotes erroneous data. dtype: float32 - long_name: maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag - hourly_minimum_downward_shortwave_irradiance_at_earth_surface: - description: Minimum global solar radiation + long_name: sur_temp_max_flag + maximum_solar_irradiance: + description: Maximum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI units at the Earth surface over specified period. dtype: float32 - flag: solarad_min_flag - long_name: minimum_downward_shortwave_irradiance_at_earth_surface + long_name: solarad_max + quality_flag: solarad_max_flag units: W m-2 - hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for minimum global solar radiation. 0 denotes good data and 3 denotes erroneous data. + maximum_solar_irradiance_quality_flag: + description: Quality indicator for the variable 'maximum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data. dtype: float32 - long_name: minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag - downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for average global solar radiation. 0 denotes good data and 3 denotes erroneous data. + long_name: solarad_max_flag + minimum_air_temperature: + description: "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: downward_shortwave_irradiance_at_earth_surface - location_latitude: - description: Latitude of the station (deg. North) + long_name: t_min + min_negative_total_uncertainty: tminerr_m + min_positive_total_uncertainty: tminerr_p + units: K + minimum_air_temperature_negative_total_uncertainty: + description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: latitude - units: degree_north - location_longitude: - description: Longitude of the station (deg. East) + long_name: tminerr_m + units: K + minimum_air_temperature_positive_total_uncertainty: + description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: longitude - units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn + long_name: tminerr_p + units: K + minimum_soil_temperature: + description: From measurements of "soil temperature" (described in this list), this parameter provides the lowest value measured during the aggregation period. + dtype: float32 + long_name: sur_temp_min + quality_flag: sur_temp_min_flag + units: K + minimum_soil_temperature_quality_flag: + description: This parameter indicates if the "Minimum soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). + dtype: float32 + long_name: sur_temp_min_flag + minimum_solar_irradiance: + description: Minimum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI unitsat the Earth surface. + dtype: float32 + long_name: solarad_min + quality_flag: solarad_min_flag + units: W m-2 + minimum_solar_irradiance_quality_flag: + description: Quality indicator for variable 'minimum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: solarad_min_flag relative_humidity: - description: Hourly relative humidity is computed from 5-minute averages in almost all cases, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. + description: This parameter is the ratio of the amount of atmospheric moisture present in the air relative to the amount that would be present if the air were saturated with respect to water or ice. This ratio is multiplied by 100 to provide numbers in percent (%). The measurements are taken at a height of 2 metres above the ground surface. The instrument is a capacitive thin-film device. It measures the flow of electricity across two electrodes separated by a polymer film. The capacitance of the film is converted to relative humidity by a calibration equation. The values are averaged over 5-minute periods, obtained by selecting "sub-hourly" aggregation period. Other aggregation periods ("hourly", "daily") are also available, in which case the results are based on further averages of the 5-minute data. A quality flag is also available for this variable. dtype: float32 - flag: rh_hr_avg_flag - long_name: average_relative_humidity + long_name: rh_hr_avg + quality_flag: rh_hr_avg_flag units: '%' relative_humidity_quality_flag: - description: QC flag for RH average. 0 denotes good data and 3 denotes erroneous data. + description: This parameter indicates if the "Relative humidity" (described in this list) may be used because it is based on an average of good data (value of 0), or if it should be treated with suspicion because erroneous data were detected (value of 3). dtype: float32 - long_name: average_relative_humidity_quality_flag + long_name: rh_hr_avg_flag report_id: - description: Identifier in the USCRN meta-database + description: Link to header information. + dtype: float32 long_name: id report_timestamp: - description: observation date time UTC + description: This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0). dtype: datetime64[ns] long_name: date_of_observation soil_moisture_100cm_from_earth_surface: - description: Average soil moisture at 100 cm below the surface + description: Average content of liquid water in a surface soil layer of 0 to 100 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_100cm_from_earth_surface + long_name: soil_moisture_100 units: m3 m-3 soil_moisture_10cm_from_earth_surface: - description: Average soil moisture at 10 cm below the surface + description: Average content of liquid water in a surface soil layer of 0 to 10 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_10cm_from_earth_surface + long_name: soil_moisture_10 units: m3 m-3 soil_moisture_20cm_from_earth_surface: - description: Average soil moisture at 20 cm below the surface + description: Average content of liquid water in a surface soil layer of 0 to 20 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_20cm_from_earth_surface + long_name: soil_moisture_20 units: m3 m-3 soil_moisture_50cm_from_earth_surface: - description: Average soil moisture at 50 cm below the surface + description: Average content of liquid water in a surface soil layer of 0 to 50 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_50cm_from_earth_surface + long_name: soil_moisture_50 units: m3 m-3 soil_moisture_5cm_from_earth_surface: - description: Average soil moisture at 5 cm below the surface + description: Average content of liquid water in a surface soil layer of 0 to 5 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_5cm_from_earth_surface + long_name: soil_moisture_5 units: m3 m-3 soil_temperature: - description: On 2013-01-07 at 1500 UTC, USCRN began reporting corrected surface temperature measurements for some stations. These changes impact previous users of the data because the corrected values differ from uncorrected values. To distinguish between uncorrected (raw) and corrected surface temperature measurements, a surface temperature type field was added to the data product. The possible values of the this field are "R" to denote raw surface temperature measurements, "C" to denote corrected surface temperature measurements, and "U" for unknown/missing. + description: Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period. dtype: float32 - flag: sur_temp_flag - long_name: average_soil_temperature + long_name: sur_temp processing_level: sur_temp_type + quality_flag: sur_temp_flag units: K soil_temperature_100cm_from_earth_surface: - description: Average soil temperature at 100 cm below the surface + description: Average soil temperature measured at 100 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_100cm_from_earth_surface + long_name: soil_temp_100 units: K soil_temperature_10cm_from_earth_surface: - description: Average soil temperature at 10 cm below the surface + description: Average soil temperature measured at 10 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_10cm_from_earth_surface + long_name: soil_temp_10 units: K soil_temperature_20cm_from_earth_surface: - description: Average soil temperature at 20 cm below the surface + description: Average soil temperature measured at 20 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_20cm_from_earth_surface + long_name: soil_temp_20 units: K soil_temperature_50cm_from_earth_surface: - description: Average soil temperature at 50 cm below the surface + description: Average soil temperature measured at 50 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_50cm_from_earth_surface + long_name: soil_temp_50 units: K soil_temperature_5cm_from_earth_surface: - description: Average soil temperature at 5 cm below the surface - dtype: float32 - long_name: soil_temperature_5cm_from_earth_surface - units: K - hourly_maximum_soil_temperature: - description: Maximum infrared surface temperature + description: Average soil temperature measured at 5 cm below the surface level over the aggregation period. dtype: float32 - flag: sur_temp_max_flag - long_name: maximum_soil_temperature + long_name: soil_temp_5 units: K - hourly_maximum_soil_temperature_flag: - description: QC flag for infrared surface temperature maximum. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: maximum_soil_temperature_quality_flag - hourly_minimum_soil_temperature: - description: Minimum infrared surface temperature - dtype: float32 - flag: sur_temp_min_flag - long_name: minimum_soil_temperature - units: K - hourly_minimum_soil_temperature_quality_flag: - description: QC flag for infrared surface temperature minimum. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: minimum_soil_temperature_quality_flag soil_temperature_processing_level: - description: Type of infrared surface temperature measurement + description: 'This parameter indicates the level of processing applied to the soil temperature measurement: raw data (''R''), corrected data (''C''). A letter ''U'' indicates that this information is unknown.' dtype: float32 - long_name: soil_temperature_processing_level + long_name: sur_temp_type soil_temperature_quality_flag: - description: QC flag for infrared surface temperature. 0 denotes good data and 3 denotes erroneous data. + description: This parameter indicates if the "Soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). dtype: float32 - long_name: average_soil_temperature_quality_flag + long_name: sur_temp_flag + solar_irradiance: + description: The quantity with standard name solar_irradiance, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. "Irradiance" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy. + dtype: float32 + long_name: solarad + quality_flag: solarad_flag + units: W m-2 + solar_irradiance_quality_flag: + description: Quality indicator variable 'solar_irradiance'. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: solarad_flag primary_station_id: - description: Station identification code + description: This is the station identification code. dtype: object long_name: wbanno mandatory_columns: @@ -778,7 +919,7 @@ sources: - columns: - sur_temp_type group_name: processing_level - USCRN_MONTHLY: + uscrn_monthly: cdm_mapping: melt_columns: true rename: @@ -793,12 +934,14 @@ sources: random_m: air_temperature_negative_random_uncertainty random_p: air_temperature_positive_random_uncertainty sitename: alternative_name + solrad_monthly_avg: monthly_global_solar_radiation sur_temp_monthly_avg: soil_temperature sur_temp_monthly_max: monthly_maximum_soil_temperature sur_temp_monthly_min: monthly_minimum_soil_temperature sur_temp_monthly_type: soil_temperature_processing_level sys_m: air_temperature_negative_systematic_uncertainty sys_p: air_temperature_positive_systematic_uncertainty + t: air_temperature t_monthly_max: daily_maximum_air_temperature t_monthly_mean: air_temperature t_monthly_min: daily_minimum_air_temperature @@ -818,136 +961,155 @@ sources: data_table: unc_monthly descriptions: accumulated_precipitation: - description: The total amount of precipitation + description: This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area. dtype: float32 - long_name: accumulated_precipitation + long_name: p_monthly_calc units: mm air_temperature: - description: The mean air temperature calculated using the typical historical approach of (T_MONTHLY_MAX + T_MONTHLY_MIN) / 2 - dtype: float32 - long_name: mean_air_temperature - units: K - air_temperature_max_negative_total_uncertainty: - description: negative uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_negative_total_uncertainty - units: K - air_temperature_max_positive_total_uncertainty: - description: positive uncertainties in tmax which are related to temperature + description: "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: air_temperature_max_positive_total_uncertainty - units: K - air_temperature_min_negative_total_uncertainty: - description: negative uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_negative_total_uncertainty - units: K - air_temperature_min_positive_total_uncertainty: - description: positive uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_positive_total_uncertainty + long_name: t + negative_quasisystematic_uncertainty: quasisys_m + negative_random_uncertainty: random_m + negative_systematic_uncertainty: sys_m + negative_total_uncertainty: terr_m + positive_quasisystematic_uncertainty: quasisys_p + positive_random_uncertainty: random_p + positive_systematic_uncertainty: sys_p + positive_total_uncertainty: terr_p units: K air_temperature_negative_quasisystematic_uncertainty: - description: -ve/-ve Quasi-systematic + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_quasisystematic_uncertainty + long_name: quasisys_m units: K air_temperature_negative_random_uncertainty: - description: -ve Random uncertainty + description: "For \"Air temperature\" (described in this list), this parameter one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_random_uncertainty + long_name: random_m units: K air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty + long_name: sys_m units: K air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_total_uncertainty + long_name: terr_m units: K air_temperature_positive_quasisystematic_uncertainty: - description: +ve Quasi-systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_quasisystematic_uncertainty + long_name: quasisys_p units: K air_temperature_positive_random_uncertainty: - description: +ve Random uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_random_uncertainty + long_name: random_p units: K air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty + long_name: sys_p units: K air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_total_uncertainty + long_name: terr_p units: K alternative_name: - description: Alternative name for station + description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object long_name: sitename - daily_maximum_air_temperature: - description: The maximum air temperature + latitude: + description: Latitude of the measurement station, -90 to 90. + dtype: float32 + long_name: latitude + units: degree_north + logbook_version: + description: LogBook software and version. + dtype: float32 + long_name: crx_vn_monthly + longitude: + description: Longitude of the measurement station, -180.0 to 180.0. dtype: float32 - long_name: air_temperature_max + long_name: longitude + units: degree_east + maximum_air_temperature: + description: "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." + dtype: float32 + long_name: t_monthly_max negative_total_uncertainty: tmaxerr_m positive_total_uncertainty: tmaxerr_p units: K - daily_minimum_air_temperature: - description: The minimum air temperature + maximum_air_temperature_negative_total_uncertainty: + description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + dtype: float32 + long_name: tmaxerr_m + units: K + maximum_air_temperature_positive_total_uncertainty: + description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + dtype: float32 + long_name: tmaxerr_p + units: K + maximum_soil_temperature: + description: From measurements of "soil temperature" (described in this list), this parameter provides the highest value measured during the aggregation period. + dtype: float32 + long_name: sur_temp_monthly_max + units: K + mean_air_temperature: + description: "This parameter is the average of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." + dtype: float32 + long_name: t_monthly_mean + units: K + minimum_air_temperature: + description: "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: minimum_air_temperature + long_name: t_monthly_min negative_total_uncertainty: tminerr_m positive_total_uncertainty: tminerr_p units: K - location_latitude: - description: Station latitude, using WGS-84, with a precision of 4 decimal places + minimum_air_temperature_negative_total_uncertainty: + description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: latitude - units: degree_north - location_longitude: - description: Station longitude, using WGS-84, with a precision of 4 decimal places + long_name: tminerr_m + units: K + minimum_air_temperature_positive_total_uncertainty: + description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: longitude - units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn_monthly + long_name: tminerr_p + units: K + minimum_soil_temperature: + description: From measurements of "soil temperature" (described in this list), this parameter provides the lowest value measured during the aggregation period. + dtype: float32 + long_name: sur_temp_monthly_min + units: K + monthly_global_solar_radiation: + description: This parameter is the total solar energy(direct and diffuse) measured at the Earth's surface per unit of area. in MJ/meter^2, calculated from the daily average global solar radiation rates and converted to energy by integrating over time. For simplicity of use, the values are provided in Mega-Joules per square metre (MJ/m**2). + dtype: float32 + long_name: solrad_monthly_avg + units: MJ m-2 report_id: - description: Identifier in the USCRN meta-database + description: Link to header information. + dtype: float32 long_name: id report_timestamp: - description: observation date time UTC + description: This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0). dtype: datetime64[ns] long_name: date_of_observation soil_temperature: - description: Monthly maximum/minimum/average surface temperatures are the average of all available daily max/min/avg values. To be considered valid, there must be fewer than 4 consecutive values missing, and no more than 5 total values missing. + description: Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period. dtype: float32 - long_name: average_soil_temperature + long_name: sur_temp_monthly_avg processing_level: sur_temp_monthly_type units: K - monthly_maximum_soil_temperature: - description: The maximum infrared surface temperature - dtype: float32 - long_name: maximum_soil_temperature - units: K - monthly_minimum_soil_temperature: - description: The minimum infrared surface temperature - dtype: float32 - long_name: minimum_soil_temperature - units: K soil_temperature_processing_level: - description: Type of infrared surface temperature measurement + description: 'This parameter indicates the level of processing applied to the soil temperature measurement: raw data (''R''), corrected data (''C''). A letter ''U'' indicates that this information is unknown.' dtype: float32 - long_name: soil_temperature_processing_level + long_name: sur_temp_monthly_type primary_station_id: - description: Station identification code + description: This is the station identification code. dtype: object long_name: wbanno mandatory_columns: @@ -961,10 +1123,12 @@ sources: - report_timestamp products: - columns: + - t - t_monthly_max - t_monthly_min - t_monthly_mean - p_monthly_calc + - solrad_monthly_avg - sur_temp_monthly_max - sur_temp_monthly_min - sur_temp_monthly_avg @@ -1000,7 +1164,7 @@ sources: - columns: - sur_temp_monthly_type group_name: processing_level - USCRN_SUBHOURLY: + uscrn_subhourly: cdm_mapping: melt_columns: true rename: @@ -1017,8 +1181,8 @@ sources: sitename: alternative_name soil_moisture_5: soil_moisture_5cm_from_earth_surface soil_temperature_5: soil_temperature_5cm_from_earth_surface - solar_radiation: downward_shortwave_irradiance_at_earth_surface - sr_flag: downward_shortwave_irradiance_at_earth_surface_quality_flag + solar_radiation: solar_irradiance + sr_flag: solar_irradiance_quality_flag st_flag: soil_temperature_processing_level_quality_flag st_type: soil_temperature_processing_level surface_temperature: soil_temperature @@ -1028,8 +1192,8 @@ sources: terr_m: air_temperature_negative_total_uncertainty terr_p: air_temperature_positive_total_uncertainty wbanno: primary_station_id - wetness: wetness wet_flag: wetness_quality_flag + wetness: wetness wind_1_5: wind_speed_2_meters_from_earth_surface wind_flag: wind_speed_2_meters_from_earth_surface_quality_flag unit_changes: @@ -1040,124 +1204,129 @@ sources: scale: 1 data_table: unc_subhourly descriptions: + 2m_wind_speed: + description: This parameter is the horizontal velocity of the air near the surface. It is measured using a 3-cup anemometer placed at the same height at the air temperature shield intake. The exact measurement height is 1.5 metres above the ground surface. + dtype: float32 + long_name: wind_1_5 + quality_flag: wind_flag + units: m s-1 + 2m_wind_speed_quality_flag: + description: This parameter indicates if the "2m wind speed" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (non-zero value). + dtype: float32 + long_name: wind_flag accumulated_precipitation: - description: Total amount of precipitation + description: This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area. dtype: float32 - long_name: accumulated_precipitation + long_name: precipitation units: mm air_temperature: - description: air temperature + description: "This parameter is the temperature of the air measured at the station, typically at about 1.5 metres above the ground surface. This parameter is the mean of measurements made at 10-second intervals during the time aggregation chosen by the user. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Several error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 - long_name: average_air_temperature + long_name: t negative_systematic_uncertainty: sys_m negative_total_uncertainty: terr_m positive_systematic_uncertainty: sys_p positive_total_uncertainty: terr_p - quasisystematic_uncertainty: quasisys_pm random_uncertainty: random_pm units: K air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty + long_name: sys_m units: K air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_negative_total_uncertainty + long_name: terr_m units: K air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive systematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty + long_name: sys_p units: K air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature - dtype: float32 - long_name: air_temperature_positive_total_uncertainty - units: K - air_temperature_quasisystematic_uncertainty: - description: +ve/-ve Quasi-systematic + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_quasisystematic_uncertainty + long_name: terr_p units: K air_temperature_random_uncertainty: - description: +ve/-ve Random + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Random uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 - long_name: air_temperature_random_uncertainty + long_name: random_pm units: K alternative_name: - description: Alternative name for station + description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object long_name: sitename - downward_shortwave_irradiance_at_earth_surface: - description: Average global solar radiation received - dtype: float32 - flag: sr_flag - long_name: downward_shortwave_irradiance_at_earth_surface - units: W m-2 - downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for the average global solar radiation measurement - dtype: float32 - long_name: downward_shortwave_irradiance_at_earth_surface_quality_flag - location_latitude: - description: Latitude of the station (deg. North) + latitude: + description: Latitude of the measurement station, -90 to 90. dtype: float32 long_name: latitude units: degree_north - location_longitude: - description: Longitude of the station (deg. East) + logbook_version: + description: LogBook software and version. + dtype: float32 + long_name: crx_vn + longitude: + description: Longitude of the measurement station, -180.0 to 180.0. dtype: float32 long_name: longitude units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn relative_humidity: - description: All USCRN stations now report 5-minute relative humidity averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. + description: This parameter is the ratio of the amount of atmospheric moisture present in the air relative to the amount that would be present if the air were saturated with respect to water or ice. This ratio is multiplied by 100 to provide numbers in percent (%). The measurements are taken at a height of 2 metres above the ground surface. The instrument is a capacitive thin-film device. It measures the flow of electricity across two electrodes separated by a polymer film. The capacitance of the film is converted to relative humidity by a calibration equation. The values are averaged over 5-minute periods, obtained by selecting "sub-hourly" aggregation period. Other aggregation periods ("hourly", "daily") are also available, in which case the results are based on further averages of the 5-minute data. A quality flag is also available for this variable. dtype: float32 - flag: rh_flag - long_name: average_relative_humidity + long_name: relative_humidity + quality_flag: rh_flag units: '%' relative_humidity_quality_flag: - description: QC flag for the relative humidity measurement + description: This parameter indicates if the "Relative humidity" (described in this list) may be used because it is based on an average of good data (value of 0), or if it should be treated with suspicion because erroneous data were detected (value of 3). dtype: float32 - long_name: average_relative_humidity_quality_flag + long_name: rh_flag report_id: - description: Identifier in the USCRN meta-database + description: Link to header information. + dtype: float32 long_name: id report_timestamp: - description: observation date time UTC + description: This parameter provides the observation date and time, including time zone information with respect to UTC (e.g. 1991-01-01 12:00:00+0). dtype: datetime64[ns] long_name: date_of_observation soil_moisture_5cm_from_earth_surface: - description: Average soil moisture at 5 cm below the surface + description: Average content of liquid water in a surface soil layer of 0 to 5 cm depth expressed as m3 water per m3 soil aggregation period. dtype: float32 - long_name: soil_moisture_5cm_from_earth_surface + long_name: soil_moisture_5 units: m3 m-3 soil_temperature: - description: USCRN stations have multiple co-located soil sensors that record independent measurements. The soil values reported in this dataset are calculated from these multiple independent measurements. Soil moisture is the ratio of water volume over sample volume (m^3 water m^-3 soil). + description: Average infrared temperature of the soil measured using infrared thermal imaging technology, over the aggregation period. dtype: float32 - flag: st_flag - long_name: average_soil_temperature + long_name: surface_temperature processing_level: st_type + quality_flag: st_flag units: K soil_temperature_5cm_from_earth_surface: - description: Average soil temperature at 5 cm below the surface + description: Average soil temperature measured at 5 cm below the surface level over the aggregation period. dtype: float32 - long_name: soil_temperature_5cm_from_earth_surface + long_name: soil_temperature_5 units: K soil_temperature_processing_level: - description: The type of infrared surface temperature measurement + description: 'This parameter indicates the level of processing applied to the soil temperature measurement: raw data (''R''), corrected data (''C''). A letter ''U'' indicates that this information is unknown.' dtype: float32 - long_name: soil_temperature_processing_level + long_name: st_type soil_temperature_processing_level_quality_flag: - description: QC flag for the surface temperature measurement + description: A value of 0 in this parameter indicates that the data processing of soil temperature did not return any error. dtype: float32 - long_name: soil_temperature_processing_level_quality_flag + long_name: st_flag + solar_irradiance: + description: The quantity with standard name solar_irradiance, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. "Irradiance" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy. + dtype: float32 + long_name: solar_radiation + quality_flag: sr_flag + units: W m-2 + solar_irradiance_quality_flag: + description: Quality indicator variable 'solar_irradiance'. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: sr_flag primary_station_id: - description: Station identification code + description: This is the station identification code. dtype: object long_name: wbanno wetness: @@ -1167,19 +1336,9 @@ sources: quality_flag: wet_flag units: Ohms wetness_quality_flag: - description: QC flag for the wetness measurement - dtype: float32 - long_name: wetness_quality_flag - wind_speed_2_meters_from_earth_surface: - description: Average wind speed, in meters per second, at a height of 1.5 meters - dtype: float32 - flag: wind_flag - long_name: wind_speed_2_meters_from_earth_surface - units: m s-1 - wind_speed_2_meters_from_earth_surface_quality_flag: - description: QC flag for the wind speed measurement + description: This parameter indicates if the "Wetness" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (non-zero value). dtype: float32 - long_name: wind_speed_2_meters_from_earth_surface_quality_flag + long_name: wet_flag mandatory_columns: - alternative_name - primary_station_id @@ -1210,6 +1369,9 @@ sources: - columns: - terr_p group_name: positive_total_uncertainty + - columns: + - terr_m + group_name: negative_total_uncertainty - columns: - quasisys_pm group_name: quasisystematic_uncertainty diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml new file mode 100644 index 0000000..f154064 --- /dev/null +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml @@ -0,0 +1,1235 @@ +global_attributes: + contactemail: https://support.ecmwf.int + licence_list: 20180314_Copernicus_License_V1.1 + responsible_organisation: ECMWF +out_columns_order: +- primary_station_id +- alternative_name +- report_timestamp +- report_id +- logbook_version +- longitude +- latitude +- air_temperature +- air_temperature_positive_random_uncertainty +- air_temperature_positive_systematic_uncertainty +- air_temperature_positive_quasisystematic_uncertainty +- air_temperature_positive_total_uncertainty +- air_temperature_negative_random_uncertainty +- air_temperature_negative_systematic_uncertainty +- air_temperature_negative_quasisystematic_uncertainty +- air_temperature_negative_total_uncertainty +- air_temperature_random_uncertainty +- air_temperature_quasisystematic_uncertainty +- air_temperature_max +- air_temperature_max_negative_total_uncertainty +- air_temperature_max_positive_total_uncertainty +- air_temperature_min +- air_temperature_min_negative_total_uncertainty +- air_temperature_min_positive_total_uncertainty +- air_temperature_mean +- air_temperature_mean_negative_total_uncertainty +- air_temperature_mean_positive_total_uncertainty +- relative_humidity +- relative_humidity_quality_flag +- relative_humidity_max +- relative_humidity_min +- soil_temperature +- soil_temperature_quality_flag +- soil_temperature_max +- soil_temperature_max_quality_flag +- soil_temperature_min +- soil_temperature_min_quality_flag +- downward_shortwave_irradiance_at_earth_surface +- downward_shortwave_irradiance_at_earth_surface_quality_flag +- downward_shortwave_irradiance_at_earth_surface_max +- downward_shortwave_irradiance_at_earth_surface_max_quality_flag +- downward_shortwave_irradiance_at_earth_surface_min +- downward_shortwave_irradiance_at_earth_surface_min_quality_flag +- soil_moisture_5cm_from_earth_surface +- soil_moisture_10cm_from_earth_surface +- soil_moisture_20cm_from_earth_surface +- soil_moisture_50cm_from_earth_surface +- soil_moisture_100cm_from_earth_surface +- soil_temperature_5cm_from_earth_surface +- soil_temperature_10cm_from_earth_surface +- soil_temperature_20cm_from_earth_surface +- soil_temperature_50cm_from_earth_surface +- soil_temperature_100cm_from_earth_surface +- soil_temperature_processing_level +- soil_temperature_processing_level_quality_flag +- wind_speed_2_meters_from_earth_surface +- wind_speed_2_meters_from_earth_surface_quality_flag +- wetness +- wetness_quality_flag +- accumulated_precipitation +- daily_global_solar_radiation_rates +- monthly_global_solar_radiation_rates +products_hierarchy: +- variables +- flag +- processing_level +- positive_total_uncertainty +- negative_total_uncertainty +- max_positive_total_uncertainty +- max_negative_total_uncertainty +- min_positive_total_uncertainty +- min_negative_total_uncertainty +- mean_positive_total_uncertainty +- mean_negative_total_uncertainty +- random_uncertainty +- positive_random_uncertainty +- negative_random_uncertainty +- positive_systematic_uncertainty +- negative_systematic_uncertainty +- quasisystematic_uncertainty +- positive_quasisystematic_uncertainty +- negative_quasisystematic_uncertainty +sources: + USCRN_DAILY: + cdm_mapping: + melt_columns: true + rename: + crx_vn: logbook_version + date_of_observation: report_timestamp + id: report_id + latitude: latitude|station_configuration + longitude: longitude|station_configuration + p_daily_calc: accumulated_precipitation + quasisys_m: air_temperature_negative_quasisystematic_uncertainty + quasisys_p: air_temperature_positive_quasisystematic_uncertainty + random_m: air_temperature_negative_random_uncertainty + random_p: air_temperature_positive_random_uncertainty + rh_daily_avg: relative_humidity + rh_daily_max: daily_maximum_relative_humidity + rh_daily_min: daily_minimum_relative_humidity + sitename: alternative_name + soil_moisture_100_daily: soil_moisture_100cm_from_earth_surface + soil_moisture_10_daily: soil_moisture_10cm_from_earth_surface + soil_moisture_20_daily: soil_moisture_20cm_from_earth_surface + soil_moisture_50_daily: soil_moisture_50cm_from_earth_surface + soil_moisture_5_daily: soil_moisture_5cm_from_earth_surface + soil_temp_100_daily: soil_temperature_100cm_from_earth_surface + soil_temp_10_daily: soil_temperature_10cm_from_earth_surface + soil_temp_20_daily: soil_temperature_20cm_from_earth_surface + soil_temp_50_daily: soil_temperature_50cm_from_earth_surface + soil_temp_5_daily: soil_temperature_5cm_from_earth_surface + sur_temp_daily_avg: soil_temperature + sur_temp_daily_type: soil_temperature_processing_level + sys_m: air_temperature_negative_systematic_uncertainty + sys_p: air_temperature_positive_systematic_uncertainty + t: air_temperature + t_daily_max: daily_maximum_air_temperature + t_daily_mean: daily_mean_air_temperature + t_daily_min: daily_minimum_air_temperature + terr_m: air_temperature_negative_total_uncertainty + terr_p: air_temperature_positive_total_uncertainty + tmaxerr_m: air_temperature_max_negative_total_uncertainty + tmaxerr_p: air_temperature_max_positive_total_uncertainty + tmeanerr_m: air_temperature_mean_negative_total_uncertainty + tmeanerr_p: air_temperature_mean_positive_total_uncertainty + tminerr_m: air_temperature_min_negative_total_uncertainty + tminerr_p: air_temperature_min_positive_total_uncertainty + wbanno: primary_station_id + unit_changes: + accumulated_precipitation: + names: + mm: mm + offset: 0 + scale: 1 + data_table: unc_daily + descriptions: + accumulated_precipitation: + description: Total amount of precipitation + dtype: float32 + long_name: accumulated_precipitation + units: mm + air_temperature: + description: 'Mean air temperature calculated using the typical historical approach: (T_DAILY_MAX + T_DAILY_MIN) / 2' + dtype: float32 + long_name: mean_air_temperature + negative_total_uncertainty: tmeanerr_m + positive_total_uncertainty: tmeanerr_p + units: K + air_temperature_max_negative_total_uncertainty: + description: negative uncertainties in tmax which are related to temperature + dtype: float32 + long_name: air_temperature_max_negative_total_uncertainty + units: K + air_temperature_max_positive_total_uncertainty: + description: positive uncertainties in tmax which are related to temperature + dtype: float32 + long_name: air_temperature_max_positive_total_uncertainty + units: K + air_temperature_mean_negative_total_uncertainty: + description: tmeanerr_m, tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (negative per using the Hourly data) + dtype: float32 + long_name: mean_air_temperature_negative_total_uncertainty + units: K + air_temperature_mean_positive_total_uncertainty: + description: tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (positive part using the Hourly data) + dtype: float32 + long_name: mean_air_temperature_positive_total_uncertainty + units: K + air_temperature_min_negative_total_uncertainty: + description: negative uncertainties in tmin which are related to temperature + dtype: float32 + long_name: minimum_air_temperature_negative_total_uncertainty + units: K + air_temperature_min_positive_total_uncertainty: + description: positive uncertainties in tmin which are related to temperature + dtype: float32 + long_name: minimum_air_temperature_positive_total_uncertainty + units: K + air_temperature_negative_quasisystematic_uncertainty: + description: -ve/-ve Quasi-systematic + dtype: float32 + long_name: air_temperature_negative_quasisystematic_uncertainty + units: K + air_temperature_negative_random_uncertainty: + description: -ve Random uncertainty + dtype: float32 + long_name: air_temperature_negative_random_uncertainty + units: K + air_temperature_negative_systematic_uncertainty: + description: -ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_negative_systematic_uncertainty + units: K + air_temperature_negative_total_uncertainty: + description: negative uncertainty of temperature + dtype: float32 + long_name: air_temperature_negative_total_uncertainty + units: K + air_temperature_positive_quasisystematic_uncertainty: + description: +ve Quasi-systematic uncertainty + dtype: float32 + long_name: air_temperature_positive_quasisystematic_uncertainty + units: K + air_temperature_positive_random_uncertainty: + description: +ve Random uncertainty + dtype: float32 + long_name: air_temperature_positive_random_uncertainty + units: K + air_temperature_positive_systematic_uncertainty: + description: +ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_positive_systematic_uncertainty + units: K + air_temperature_positive_total_uncertainty: + description: positive uncertainty of temperature + dtype: float32 + long_name: air_temperature_positive_total_uncertainty + units: K + alternative_name: + description: Alternative name for station + dtype: object + long_name: sitename + daily_maximum_air_temperature: + description: Maximum air temperature + dtype: float32 + long_name: air_temperature_max + negative_total_uncertainty: tmaxerr_m + positive_total_uncertainty: tmaxerr_p + units: K + daily_minimum_air_temperature: + description: Minimum air temperature + dtype: float32 + long_name: minimum_air_temperature + negative_total_uncertainty: tminerr_m + positive_total_uncertainty: tminerr_p + units: K + location_latitude: + description: Latitude of the station (deg. North) + dtype: float32 + long_name: latitude + units: degree_north + location_longitude: + description: Longitude of the station (deg. East) + dtype: float32 + long_name: longitude + units: degree_east + logbook_version: + description: The version number of the station datalogger program that was in effect at the time of the observation + dtype: object + long_name: crx_vn + relative_humidity: + description: Relative humidity is computed from 5-minute values in almost all cases. All USCRN stations now report 5-minute averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. + dtype: float32 + long_name: average_relative_humidity + units: '%' + daily_maximum_relative_humidity: + description: Maximum relative humidity + dtype: float32 + long_name: maximum_relative_humidity + units: '%' + daily_minimum_relative_humidity: + description: Minimum relative humidity + dtype: float32 + long_name: minimum_relative_humidity + units: '%' + report_id: + description: Identifier in the USCRN meta-database + long_name: id + report_timestamp: + description: observation date time UTC + dtype: datetime64[ns] + long_name: date_of_observation + soil_moisture_100cm_from_earth_surface: + description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 100 cm below the surface. + dtype: float32 + long_name: soil_moisture_100cm_from_earth_surface + units: m3 m-3 + soil_moisture_10cm_from_earth_surface: + description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 10 cm below the surface. + dtype: float32 + long_name: soil_moisture_10cm_from_earth_surface + units: m3 m-3 + soil_moisture_20cm_from_earth_surface: + description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 20 cm below the surface. + dtype: float32 + long_name: soil_moisture_20cm_from_earth_surface + units: m3 m-3 + soil_moisture_50cm_from_earth_surface: + description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 50 cm below the surface. + dtype: float32 + long_name: soil_moisture_50cm_from_earth_surface + units: m3 m-3 + soil_moisture_5cm_from_earth_surface: + description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 5 cm below the surface. + dtype: float32 + long_name: soil_moisture_5cm_from_earth_surface + units: m3 m-3 + soil_temperature: + description: Average infrared surface temperature + dtype: float32 + long_name: average_soil_temperature + processing_level: sur_temp_daily_type + units: K + soil_temperature_100cm_from_earth_surface: + description: Average soil temperature, in degrees C, at 100 cm below the surface. + dtype: float32 + long_name: soil_temperature_100cm_from_earth_surface + units: K + soil_temperature_10cm_from_earth_surface: + description: Average soil temperature, in degrees C, at 10 cm below the surface. + dtype: float32 + long_name: soil_temperature_10cm_from_earth_surface + units: K + soil_temperature_20cm_from_earth_surface: + description: Average soil temperature, in degrees C, at 20 cm below the surface. + dtype: float32 + long_name: soil_temperature_20cm_from_earth_surface + units: K + soil_temperature_50cm_from_earth_surface: + description: Average soil temperature, in degrees C, at 50 cm below the surface. + dtype: float32 + long_name: soil_temperature_50cm_from_earth_surface + units: K + soil_temperature_5cm_from_earth_surface: + description: Average soil temperature, in degrees C, at 5 cm below the surface. + dtype: float32 + long_name: soil_temperature_5cm_from_earth_surface + units: K + soil_temperature_processing_level: + description: Type of infrared surface temperature measurement. + dtype: float32 + long_name: soil_temperature_processing_level + primary_station_id: + description: Station identification code + dtype: object + long_name: wbanno + mandatory_columns: + - alternative_name + - primary_station_id + - longitude|station_configuration + - latitude|station_configuration + - report_timestamp + order_by: + - primary_station_id + - report_timestamp + products: + - columns: + - t_daily_max + - t_daily_min + - t_daily_mean + - p_daily_calc + - sur_temp_daily_avg + - rh_daily_max + - rh_daily_min + - rh_daily_avg + - soil_moisture_5_daily + - soil_moisture_10_daily + - soil_moisture_20_daily + - soil_moisture_50_daily + - soil_moisture_100_daily + - soil_temp_5_daily + - soil_temp_10_daily + - soil_temp_20_daily + - soil_temp_50_daily + - soil_temp_100_daily + group_name: variables + - columns: + - random_p + group_name: positive_random_uncertainty + - columns: + - random_m + group_name: negative_random_uncertainty + - columns: + - terr_p + - tmaxerr_p + - tminerr_p + - tmeanerr_p + group_name: positive_total_uncertainty + - columns: + - terr_m + - tmaxerr_m + - tminerr_m + - tmeanerr_m + group_name: negative_total_uncertainty + - columns: + - quasisys_p + group_name: positive_quasisystematic_uncertainty + - columns: + - quasisys_m + group_name: negative_quasisystematic_uncertainty + - columns: + - sys_p + group_name: positive_systematic_uncertainty + - columns: + - sys_m + group_name: negative_systematic_uncertainty + - columns: + - sur_temp_daily_type + group_name: processing_level + USCRN_HOURLY: + cdm_mapping: + melt_columns: true + rename: + crx_vn: logbook_version + date_of_observation: report_timestamp + id: report_id + latitude: latitude|station_configuration + longitude: longitude|station_configuration + p_calc: accumulated_precipitation + quasisys_m: air_temperature_negative_quasisystematic_uncertainty + quasisys_p: air_temperature_positive_quasisystematic_uncertainty + random_pm: air_temperature_random_uncertainty + rh_hr_avg: relative_humidity + rh_hr_avg_flag: relative_humidity_quality_flag + sitename: alternative_name + soil_moisture_10: soil_moisture_10cm_from_earth_surface + soil_moisture_100: soil_moisture_100cm_from_earth_surface + soil_moisture_20: soil_moisture_20cm_from_earth_surface + soil_moisture_5: soil_moisture_5cm_from_earth_surface + soil_moisture_50: soil_moisture_50cm_from_earth_surface + soil_temp_10: soil_temperature_10cm_from_earth_surface + soil_temp_100: soil_temperature_100cm_from_earth_surface + soil_temp_20: soil_temperature_20cm_from_earth_surface + soil_temp_5: soil_temperature_5cm_from_earth_surface + soil_temp_50: soil_temperature_50cm_from_earth_surface + solarad: downward_shortwave_irradiance_at_earth_surface + solarad_flag: downward_shortwave_irradiance_at_earth_surface_quality_flag + solarad_max: hourly_maximum_downward_shortwave_irradiance_at_earth_surface + solarad_max_flag: hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag + solarad_min: hourly_minimum_downward_shortwave_irradiance_at_earth_surface + solarad_min_flag: hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag + sur_temp: soil_temperature + sur_temp_flag: soil_temperature_quality_flag + sur_temp_max: hourly_maximum_soil_temperature + sur_temp_max_flag: hourly_maximum_soil_temperature_flag + sur_temp_min: hourly_minimum_soil_temperature + sur_temp_min_flag: hourly_minimum_soil_temperature_quality_flag + sur_temp_type: soil_temperature_processing_level + sys_m: air_temperature_negative_systematic_uncertainty + sys_p: air_temperature_positive_systematic_uncertainty + t: air_temperature + t_max: daily_maximum_air_temperature + t_min: daily_minimum_air_temperature + terr_m: air_temperature_negative_total_uncertainty + terr_p: air_temperature_positive_total_uncertainty + tmaxerr_m: air_temperature_max_negative_total_uncertainty + tmaxerr_p: air_temperature_max_positive_total_uncertainty + tminerr_m: air_temperature_min_negative_total_uncertainty + tminerr_p: air_temperature_min_positive_total_uncertainty + wbanno: primary_station_id + unit_changes: + accumulated_precipitation: + names: + mm: mm + offset: 0 + scale: 1 + data_table: unc_hourly + descriptions: + accumulated_precipitation: + description: Total amount of precipitation, in mm, recorded during the hour + dtype: float32 + long_name: accumulated_precipitation + units: mm + air_temperature: + description: Near surface temperature + dtype: float32 + long_name: average_air_temperature + max_negative_total_uncertainty: tmaxerr_m + max_positive_total_uncertainty: tmaxerr_p + min_negative_total_uncertainty: tminerr_m + min_positive_total_uncertainty: tminerr_p + negative_quasisystematic_uncertainty: quasisys_m + negative_systematic_uncertainty: sys_m + negative_total_uncertainty: terr_m + positive_quasisystematic_uncertainty: quasisys_p + positive_systematic_uncertainty: sys_p + positive_total_uncertainty: terr_p + random_uncertainty: random_pm + units: K + air_temperature_max_negative_total_uncertainty: + description: negative uncertainties in tmax which are related to temperature + dtype: float32 + long_name: air_temperature_max_negative_total_uncertainty + units: K + air_temperature_max_positive_total_uncertainty: + description: positive uncertainties in tmax which are related to temperature + dtype: float32 + long_name: air_temperature_max_positive_total_uncertainty + units: K + air_temperature_min_negative_total_uncertainty: + description: negative uncertainties in tmin which are related to temperature + dtype: float32 + long_name: minimum_air_temperature_negative_total_uncertainty + units: K + air_temperature_min_positive_total_uncertainty: + description: positive uncertainties in tmin which are related to temperature + dtype: float32 + long_name: minimum_air_temperature_positive_total_uncertainty + units: K + air_temperature_negative_quasisystematic_uncertainty: + description: -ve/-ve Quasi-systematic + dtype: float32 + long_name: air_temperature_negative_quasisystematic_uncertainty + units: K + air_temperature_negative_systematic_uncertainty: + description: -ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_negative_systematic_uncertainty + units: K + air_temperature_negative_total_uncertainty: + description: negative uncertainty of temperature + dtype: float32 + long_name: air_temperature_negative_total_uncertainty + units: K + air_temperature_positive_quasisystematic_uncertainty: + description: +ve Quasi-systematic uncertainty + dtype: float32 + long_name: air_temperature_positive_quasisystematic_uncertainty + units: K + air_temperature_positive_systematic_uncertainty: + description: +ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_positive_systematic_uncertainty + units: K + air_temperature_positive_total_uncertainty: + description: positive uncertainty of temperature + dtype: float32 + long_name: air_temperature_positive_total_uncertainty + units: K + air_temperature_random_uncertainty: + description: +ve/-ve Random + dtype: float32 + long_name: air_temperature_random_uncertainty + units: K + alternative_name: + description: Alternative name for station + dtype: object + long_name: sitename + daily_maximum_air_temperature: + description: Maximum air temperature during the hour + dtype: float32 + long_name: air_temperature_max + units: K + daily_minimum_air_temperature: + description: Minimum air temperature during the hour + dtype: float32 + long_name: minimum_air_temperature + units: K + downward_shortwave_irradiance_at_earth_surface: + description: Average global solar radiation + dtype: float32 + flag: solarad_flag + long_name: downward_shortwave_irradiance_at_earth_surface + units: W m-2 + hourly_maximum_downward_shortwave_irradiance_at_earth_surface: + description: Maximum global solar radiation + dtype: float32 + flag: solarad_max_flag + long_name: maximum_downward_shortwave_irradiance_at_earth_surface + units: W m-2 + hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag: + description: QC flag for maximum global solar radiation. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag + hourly_minimum_downward_shortwave_irradiance_at_earth_surface: + description: Minimum global solar radiation + dtype: float32 + flag: solarad_min_flag + long_name: minimum_downward_shortwave_irradiance_at_earth_surface + units: W m-2 + hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag: + description: QC flag for minimum global solar radiation. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag + downward_shortwave_irradiance_at_earth_surface_quality_flag: + description: QC flag for average global solar radiation. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: downward_shortwave_irradiance_at_earth_surface + location_latitude: + description: Latitude of the station (deg. North) + dtype: float32 + long_name: latitude + units: degree_north + location_longitude: + description: Longitude of the station (deg. East) + dtype: float32 + long_name: longitude + units: degree_east + logbook_version: + description: The version number of the station datalogger program that was in effect at the time of the observation + dtype: object + long_name: crx_vn + relative_humidity: + description: Hourly relative humidity is computed from 5-minute averages in almost all cases, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. + dtype: float32 + flag: rh_hr_avg_flag + long_name: average_relative_humidity + units: '%' + relative_humidity_quality_flag: + description: QC flag for RH average. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: average_relative_humidity_quality_flag + report_id: + description: Identifier in the USCRN meta-database + long_name: id + report_timestamp: + description: observation date time UTC + dtype: datetime64[ns] + long_name: date_of_observation + soil_moisture_100cm_from_earth_surface: + description: Average soil moisture at 100 cm below the surface + dtype: float32 + long_name: soil_moisture_100cm_from_earth_surface + units: m3 m-3 + soil_moisture_10cm_from_earth_surface: + description: Average soil moisture at 10 cm below the surface + dtype: float32 + long_name: soil_moisture_10cm_from_earth_surface + units: m3 m-3 + soil_moisture_20cm_from_earth_surface: + description: Average soil moisture at 20 cm below the surface + dtype: float32 + long_name: soil_moisture_20cm_from_earth_surface + units: m3 m-3 + soil_moisture_50cm_from_earth_surface: + description: Average soil moisture at 50 cm below the surface + dtype: float32 + long_name: soil_moisture_50cm_from_earth_surface + units: m3 m-3 + soil_moisture_5cm_from_earth_surface: + description: Average soil moisture at 5 cm below the surface + dtype: float32 + long_name: soil_moisture_5cm_from_earth_surface + units: m3 m-3 + soil_temperature: + description: On 2013-01-07 at 1500 UTC, USCRN began reporting corrected surface temperature measurements for some stations. These changes impact previous users of the data because the corrected values differ from uncorrected values. To distinguish between uncorrected (raw) and corrected surface temperature measurements, a surface temperature type field was added to the data product. The possible values of the this field are "R" to denote raw surface temperature measurements, "C" to denote corrected surface temperature measurements, and "U" for unknown/missing. + dtype: float32 + flag: sur_temp_flag + long_name: average_soil_temperature + processing_level: sur_temp_type + units: K + soil_temperature_100cm_from_earth_surface: + description: Average soil temperature at 100 cm below the surface + dtype: float32 + long_name: soil_temperature_100cm_from_earth_surface + units: K + soil_temperature_10cm_from_earth_surface: + description: Average soil temperature at 10 cm below the surface + dtype: float32 + long_name: soil_temperature_10cm_from_earth_surface + units: K + soil_temperature_20cm_from_earth_surface: + description: Average soil temperature at 20 cm below the surface + dtype: float32 + long_name: soil_temperature_20cm_from_earth_surface + units: K + soil_temperature_50cm_from_earth_surface: + description: Average soil temperature at 50 cm below the surface + dtype: float32 + long_name: soil_temperature_50cm_from_earth_surface + units: K + soil_temperature_5cm_from_earth_surface: + description: Average soil temperature at 5 cm below the surface + dtype: float32 + long_name: soil_temperature_5cm_from_earth_surface + units: K + hourly_maximum_soil_temperature: + description: Maximum infrared surface temperature + dtype: float32 + flag: sur_temp_max_flag + long_name: maximum_soil_temperature + units: K + hourly_maximum_soil_temperature_flag: + description: QC flag for infrared surface temperature maximum. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: maximum_soil_temperature_quality_flag + hourly_minimum_soil_temperature: + description: Minimum infrared surface temperature + dtype: float32 + flag: sur_temp_min_flag + long_name: minimum_soil_temperature + units: K + hourly_minimum_soil_temperature_quality_flag: + description: QC flag for infrared surface temperature minimum. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: minimum_soil_temperature_quality_flag + soil_temperature_processing_level: + description: Type of infrared surface temperature measurement + dtype: float32 + long_name: soil_temperature_processing_level + soil_temperature_quality_flag: + description: QC flag for infrared surface temperature. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: average_soil_temperature_quality_flag + primary_station_id: + description: Station identification code + dtype: object + long_name: wbanno + mandatory_columns: + - alternative_name + - primary_station_id + - longitude|station_configuration + - latitude|station_configuration + - report_timestamp + order_by: + - primary_station_id + - report_timestamp + products: + - columns: + - t + - t_max + - t_min + - p_calc + - solarad + - solarad_max + - solarad_min + - sur_temp + - sur_temp_max + - sur_temp_min + - rh_hr_avg + - soil_moisture_5 + - soil_moisture_10 + - soil_moisture_20 + - soil_moisture_50 + - soil_moisture_100 + - soil_temp_5 + - soil_temp_10 + - soil_temp_20 + - soil_temp_50 + - soil_temp_100 + group_name: variables + - columns: + - random_pm + group_name: random_uncertainty + - columns: + - terr_p + group_name: positive_total_uncertainty + - columns: + - terr_m + group_name: negative_total_uncertainty + - columns: + - tmaxerr_p + group_name: max_positive_total_uncertainty + - columns: + - tmaxerr_m + group_name: max_negative_total_uncertainty + - columns: + - tminerr_p + group_name: min_positive_total_uncertainty + - columns: + - tminerr_m + group_name: min_negative_total_uncertainty + - columns: + - sys_p + group_name: positive_systematic_uncertainty + - columns: + - sys_m + group_name: negative_systematic_uncertainty + - columns: + - quasisys_p + group_name: positive_quasisystematic_uncertainty + - columns: + - quasisys_m + group_name: negative_quasisystematic_uncertainty + - columns: + - solarad_flag + - solarad_max_flag + - solarad_min_flag + - sur_temp_flag + - sur_temp_max_flag + - sur_temp_min_flag + - rh_hr_avg_flag + group_name: flag + - columns: + - sur_temp_type + group_name: processing_level + USCRN_MONTHLY: + cdm_mapping: + melt_columns: true + rename: + crx_vn_monthly: logbook_version + date_of_observation: report_timestamp + id: report_id + latitude: latitude|station_configuration + longitude: longitude|station_configuration + p_monthly_calc: accumulated_precipitation + quasisys_m: air_temperature_negative_quasisystematic_uncertainty + quasisys_p: air_temperature_positive_quasisystematic_uncertainty + random_m: air_temperature_negative_random_uncertainty + random_p: air_temperature_positive_random_uncertainty + sitename: alternative_name + sur_temp_monthly_avg: soil_temperature + sur_temp_monthly_max: monthly_maximum_soil_temperature + sur_temp_monthly_min: monthly_minimum_soil_temperature + sur_temp_monthly_type: soil_temperature_processing_level + sys_m: air_temperature_negative_systematic_uncertainty + sys_p: air_temperature_positive_systematic_uncertainty + t_monthly_max: daily_maximum_air_temperature + t_monthly_mean: air_temperature + t_monthly_min: daily_minimum_air_temperature + terr_m: air_temperature_negative_total_uncertainty + terr_p: air_temperature_positive_total_uncertainty + tmaxerr_m: air_temperature_max_negative_total_uncertainty + tmaxerr_p: air_temperature_max_positive_total_uncertainty + tminerr_m: air_temperature_min_negative_total_uncertainty + tminerr_p: air_temperature_min_positive_total_uncertainty + wbanno: primary_station_id + unit_changes: + accumulated_precipitation: + names: + mm: mm + offset: 0 + scale: 1 + data_table: unc_monthly + descriptions: + accumulated_precipitation: + description: The total amount of precipitation + dtype: float32 + long_name: accumulated_precipitation + units: mm + air_temperature: + description: The mean air temperature calculated using the typical historical approach of (T_MONTHLY_MAX + T_MONTHLY_MIN) / 2 + dtype: float32 + long_name: mean_air_temperature + units: K + air_temperature_max_negative_total_uncertainty: + description: negative uncertainties in tmax which are related to temperature + dtype: float32 + long_name: air_temperature_max_negative_total_uncertainty + units: K + air_temperature_max_positive_total_uncertainty: + description: positive uncertainties in tmax which are related to temperature + dtype: float32 + long_name: air_temperature_max_positive_total_uncertainty + units: K + air_temperature_min_negative_total_uncertainty: + description: negative uncertainties in tmin which are related to temperature + dtype: float32 + long_name: minimum_air_temperature_negative_total_uncertainty + units: K + air_temperature_min_positive_total_uncertainty: + description: positive uncertainties in tmin which are related to temperature + dtype: float32 + long_name: minimum_air_temperature_positive_total_uncertainty + units: K + air_temperature_negative_quasisystematic_uncertainty: + description: -ve/-ve Quasi-systematic + dtype: float32 + long_name: air_temperature_negative_quasisystematic_uncertainty + units: K + air_temperature_negative_random_uncertainty: + description: -ve Random uncertainty + dtype: float32 + long_name: air_temperature_negative_random_uncertainty + units: K + air_temperature_negative_systematic_uncertainty: + description: -ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_negative_systematic_uncertainty + units: K + air_temperature_negative_total_uncertainty: + description: negative uncertainty of temperature + dtype: float32 + long_name: air_temperature_negative_total_uncertainty + units: K + air_temperature_positive_quasisystematic_uncertainty: + description: +ve Quasi-systematic uncertainty + dtype: float32 + long_name: air_temperature_positive_quasisystematic_uncertainty + units: K + air_temperature_positive_random_uncertainty: + description: +ve Random uncertainty + dtype: float32 + long_name: air_temperature_positive_random_uncertainty + units: K + air_temperature_positive_systematic_uncertainty: + description: +ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_positive_systematic_uncertainty + units: K + air_temperature_positive_total_uncertainty: + description: positive uncertainty of temperature + dtype: float32 + long_name: air_temperature_positive_total_uncertainty + units: K + alternative_name: + description: Alternative name for station + dtype: object + long_name: sitename + daily_maximum_air_temperature: + description: The maximum air temperature + dtype: float32 + long_name: air_temperature_max + negative_total_uncertainty: tmaxerr_m + positive_total_uncertainty: tmaxerr_p + units: K + daily_minimum_air_temperature: + description: The minimum air temperature + dtype: float32 + long_name: minimum_air_temperature + negative_total_uncertainty: tminerr_m + positive_total_uncertainty: tminerr_p + units: K + location_latitude: + description: Station latitude, using WGS-84, with a precision of 4 decimal places + dtype: float32 + long_name: latitude + units: degree_north + location_longitude: + description: Station longitude, using WGS-84, with a precision of 4 decimal places + dtype: float32 + long_name: longitude + units: degree_east + logbook_version: + description: The version number of the station datalogger program that was in effect at the time of the observation + dtype: object + long_name: crx_vn_monthly + report_id: + description: Identifier in the USCRN meta-database + long_name: id + report_timestamp: + description: observation date time UTC + dtype: datetime64[ns] + long_name: date_of_observation + soil_temperature: + description: Monthly maximum/minimum/average surface temperatures are the average of all available daily max/min/avg values. To be considered valid, there must be fewer than 4 consecutive values missing, and no more than 5 total values missing. + dtype: float32 + long_name: average_soil_temperature + processing_level: sur_temp_monthly_type + units: K + monthly_maximum_soil_temperature: + description: The maximum infrared surface temperature + dtype: float32 + long_name: maximum_soil_temperature + units: K + monthly_minimum_soil_temperature: + description: The minimum infrared surface temperature + dtype: float32 + long_name: minimum_soil_temperature + units: K + soil_temperature_processing_level: + description: Type of infrared surface temperature measurement + dtype: float32 + long_name: soil_temperature_processing_level + primary_station_id: + description: Station identification code + dtype: object + long_name: wbanno + mandatory_columns: + - alternative_name + - primary_station_id + - longitude|station_configuration + - latitude|station_configuration + - report_timestamp + order_by: + - primary_station_id + - report_timestamp + products: + - columns: + - t_monthly_max + - t_monthly_min + - t_monthly_mean + - p_monthly_calc + - sur_temp_monthly_max + - sur_temp_monthly_min + - sur_temp_monthly_avg + group_name: variables + - columns: + - random_p + group_name: positive_random_uncertainty + - columns: + - random_m + group_name: negative_random_uncertainty + - columns: + - terr_p + - tmaxerr_p + - tminerr_p + group_name: positive_total_uncertainty + - columns: + - terr_m + - tmaxerr_m + - tminerr_m + group_name: negative_total_uncertainty + - columns: + - quasisys_p + group_name: positive_quasisystematic_uncertainty + - columns: + - quasisys_m + group_name: negative_quasisystematic_uncertainty + - columns: + - sys_p + group_name: positive_systematic_uncertainty + - columns: + - sys_m + group_name: negative_systematic_uncertainty + - columns: + - sur_temp_monthly_type + group_name: processing_level + USCRN_SUBHOURLY: + cdm_mapping: + melt_columns: true + rename: + crx_vn: logbook_version + date_of_observation: report_timestamp + id: report_id + latitude: latitude|station_configuration + longitude: longitude|station_configuration + precipitation: accumulated_precipitation + quasisys_pm: air_temperature_quasisystematic_uncertainty + random_pm: air_temperature_random_uncertainty + relative_humidity: relative_humidity + rh_flag: relative_humidity_quality_flag + sitename: alternative_name + soil_moisture_5: soil_moisture_5cm_from_earth_surface + soil_temperature_5: soil_temperature_5cm_from_earth_surface + solar_radiation: downward_shortwave_irradiance_at_earth_surface + sr_flag: downward_shortwave_irradiance_at_earth_surface_quality_flag + st_flag: soil_temperature_processing_level_quality_flag + st_type: soil_temperature_processing_level + surface_temperature: soil_temperature + sys_m: air_temperature_negative_systematic_uncertainty + sys_p: air_temperature_positive_systematic_uncertainty + t: air_temperature + terr_m: air_temperature_negative_total_uncertainty + terr_p: air_temperature_positive_total_uncertainty + wbanno: primary_station_id + wetness: wetness + wet_flag: wetness_quality_flag + wind_1_5: wind_speed_2_meters_from_earth_surface + wind_flag: wind_speed_2_meters_from_earth_surface_quality_flag + unit_changes: + accumulated_precipitation: + names: + mm: mm + offset: 0 + scale: 1 + data_table: unc_subhourly + descriptions: + accumulated_precipitation: + description: Total amount of precipitation + dtype: float32 + long_name: accumulated_precipitation + units: mm + air_temperature: + description: air temperature + dtype: float32 + long_name: average_air_temperature + negative_systematic_uncertainty: sys_m + negative_total_uncertainty: terr_m + positive_systematic_uncertainty: sys_p + positive_total_uncertainty: terr_p + quasisystematic_uncertainty: quasisys_pm + random_uncertainty: random_pm + units: K + air_temperature_negative_systematic_uncertainty: + description: -ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_negative_systematic_uncertainty + units: K + air_temperature_negative_total_uncertainty: + description: negative uncertainty of temperature + dtype: float32 + long_name: air_temperature_negative_total_uncertainty + units: K + air_temperature_positive_systematic_uncertainty: + description: +ve Systematic uncertainty + dtype: float32 + long_name: air_temperature_positive_systematic_uncertainty + units: K + air_temperature_positive_total_uncertainty: + description: positive uncertainty of temperature + dtype: float32 + long_name: air_temperature_positive_total_uncertainty + units: K + air_temperature_quasisystematic_uncertainty: + description: +ve/-ve Quasi-systematic + dtype: float32 + long_name: air_temperature_quasisystematic_uncertainty + units: K + air_temperature_random_uncertainty: + description: +ve/-ve Random + dtype: float32 + long_name: air_temperature_random_uncertainty + units: K + alternative_name: + description: Alternative name for station + dtype: object + long_name: sitename + downward_shortwave_irradiance_at_earth_surface: + description: Average global solar radiation received + dtype: float32 + flag: sr_flag + long_name: downward_shortwave_irradiance_at_earth_surface + units: W m-2 + downward_shortwave_irradiance_at_earth_surface_quality_flag: + description: QC flag for the average global solar radiation measurement + dtype: float32 + long_name: downward_shortwave_irradiance_at_earth_surface_quality_flag + location_latitude: + description: Latitude of the station (deg. North) + dtype: float32 + long_name: latitude + units: degree_north + location_longitude: + description: Longitude of the station (deg. East) + dtype: float32 + long_name: longitude + units: degree_east + logbook_version: + description: The version number of the station datalogger program that was in effect at the time of the observation + dtype: object + long_name: crx_vn + relative_humidity: + description: All USCRN stations now report 5-minute relative humidity averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. + dtype: float32 + flag: rh_flag + long_name: average_relative_humidity + units: '%' + relative_humidity_quality_flag: + description: QC flag for the relative humidity measurement + dtype: float32 + long_name: average_relative_humidity_quality_flag + report_id: + description: Identifier in the USCRN meta-database + long_name: id + report_timestamp: + description: observation date time UTC + dtype: datetime64[ns] + long_name: date_of_observation + soil_moisture_5cm_from_earth_surface: + description: Average soil moisture at 5 cm below the surface + dtype: float32 + long_name: soil_moisture_5cm_from_earth_surface + units: m3 m-3 + soil_temperature: + description: USCRN stations have multiple co-located soil sensors that record independent measurements. The soil values reported in this dataset are calculated from these multiple independent measurements. Soil moisture is the ratio of water volume over sample volume (m^3 water m^-3 soil). + dtype: float32 + flag: st_flag + long_name: average_soil_temperature + processing_level: st_type + units: K + soil_temperature_5cm_from_earth_surface: + description: Average soil temperature at 5 cm below the surface + dtype: float32 + long_name: soil_temperature_5cm_from_earth_surface + units: K + soil_temperature_processing_level: + description: The type of infrared surface temperature measurement + dtype: float32 + long_name: soil_temperature_processing_level + soil_temperature_processing_level_quality_flag: + description: QC flag for the surface temperature measurement + dtype: float32 + long_name: soil_temperature_processing_level_quality_flag + primary_station_id: + description: Station identification code + dtype: object + long_name: wbanno + wetness: + description: This parameter indicates the presence or absence of moisture due to precipitation, in Ohms. High values (>= 1000) indicate an absence of moisture. Low values (< 1000) indicate the presence of moisture. + dtype: float32 + long_name: wetness + quality_flag: wet_flag + units: Ohms + wetness_quality_flag: + description: QC flag for the wetness measurement + dtype: float32 + long_name: wetness_quality_flag + wind_speed_2_meters_from_earth_surface: + description: Average wind speed, in meters per second, at a height of 1.5 meters + dtype: float32 + flag: wind_flag + long_name: wind_speed_2_meters_from_earth_surface + units: m s-1 + wind_speed_2_meters_from_earth_surface_quality_flag: + description: QC flag for the wind speed measurement + dtype: float32 + long_name: wind_speed_2_meters_from_earth_surface_quality_flag + mandatory_columns: + - alternative_name + - primary_station_id + - longitude|station_configuration + - latitude|station_configuration + - report_timestamp + order_by: + - primary_station_id + - report_timestamp + products: + - columns: + - t + - precipitation + - solar_radiation + - surface_temperature + - relative_humidity + - soil_moisture_5 + - soil_temperature_5 + - wetness + - wind_1_5 + group_name: variables + - columns: + - terr_m + group_name: negative_total_uncertainty + - columns: + - random_pm + group_name: random_uncertainty + - columns: + - terr_p + group_name: positive_total_uncertainty + - columns: + - quasisys_pm + group_name: quasisystematic_uncertainty + - columns: + - sys_p + group_name: positive_systematic_uncertainty + - columns: + - sys_m + group_name: negative_systematic_uncertainty + - columns: + - sr_flag + - st_flag + - rh_flag + - wet_flag + - wind_flag + group_name: flag + - columns: + - st_type + group_name: processing_level +space_columns: + y: latitude|station_configuration + x: longitude|station_configuration diff --git a/tests/conftest.py b/tests/conftest.py index 54c5374..6353d0e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,7 +127,7 @@ def test_repository(test_session, test_s3_client, test_config): "insitu-observations-near-surface-temperature-us-climate-reference-network" ) service_definition = get_service_definition(dataset_name) - for dataset_source in ["USCRN_DAILY", "USCRN_SUBHOURLY"]: + for dataset_source in ["uscrn_daily", "uscrn_subhourly"]: start_year, end_year = get_test_years(dataset_source) run_ingestion_pipeline( dataset_name, diff --git a/tests/retrieve/test_adaptor.py b/tests/retrieve/test_adaptor.py index f7f4ebf..9af1f6c 100644 --- a/tests/retrieve/test_adaptor.py +++ b/tests/retrieve/test_adaptor.py @@ -75,10 +75,10 @@ def test_adaptor_uscrn(tmp_path): "mapping": { "remap": { "time_aggregation": { - "daily": "USCRN_DAILY", - "hourly": "USCRN_HOURLY", - "monthly": "USCRN_MONTHLY", - "sub_hourly": "USCRN_SUBHOURLY", + "daily": "uscrn_daily", + "hourly": "uscrn_hourly", + "monthly": "uscrn_monthly", + "sub_hourly": "uscrn_subhourly", }, "variable": { "maximum_air_temperature": "daily_maximum_air_temperature", diff --git a/tests/test_api.py b/tests/test_api.py index 994e165..821e7d4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -13,55 +13,55 @@ @pytest.mark.parametrize( "dataset_name,source,test_update", [ - ( - "insitu-observations-woudc-ozone-total-column-and-profiles", - "OzoneSonde", - False, - ), - ( - "insitu-observations-woudc-ozone-total-column-and-profiles", - "TotalOzone", - False, - ), - ("insitu-observations-igra-baseline-network", "IGRA", False), - ("insitu-observations-igra-baseline-network", "IGRA_H", False), - ("insitu-comprehensive-upper-air-observation-network", "CUON", True), - ("insitu-observations-gruan-reference-network", "GRUAN", False), + # ( + # "insitu-observations-woudc-ozone-total-column-and-profiles", + # "OzoneSonde", + # False, + # ), + # ( + # "insitu-observations-woudc-ozone-total-column-and-profiles", + # "TotalOzone", + # False, + # ), + # ("insitu-observations-igra-baseline-network", "IGRA", False), + # ("insitu-observations-igra-baseline-network", "IGRA_H", False), + # ("insitu-comprehensive-upper-air-observation-network", "CUON", True), + # ("insitu-observations-gruan-reference-network", "GRUAN", False), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", - "USCRN_SUBHOURLY", + "uscrn_subhourly", False, ), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", - "USCRN_HOURLY", + "uscrn_hourly", False, ), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", - "USCRN_DAILY", + "uscrn_daily", False, ), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", - "USCRN_MONTHLY", - False, - ), - ( - "insitu-observations-gnss", - "IGS", - False, - ), - ( - "insitu-observations-gnss", - "EPN", - False, - ), - ( - "insitu-observations-gnss", - "IGS_R3", + "uscrn_monthly", False, ), + # ( + # "insitu-observations-gnss", + # "IGS", + # False, + # ), + # ( + # "insitu-observations-gnss", + # "EPN", + # False, + # ), + # ( + # "insitu-observations-gnss", + # "IGS_R3", + # False, + # ), ], ) def test_run_ingestion_pipeline( diff --git a/tests/test_cdm_api.py b/tests/test_cdm_api.py index 646b275..674a655 100644 --- a/tests/test_cdm_api.py +++ b/tests/test_cdm_api.py @@ -47,7 +47,7 @@ def test_get_aux_fields_mapping_from_service_definition(): dataset_name = ( "insitu-observations-near-surface-temperature-us-climate-reference-network" ) - source = "USCRN_HOURLY" + source = "uscrn_hourly" expected = { "accumulated_precipitation": [], "air_temperature": [ @@ -223,7 +223,7 @@ def test_get_aux_vars_from_service_definition(): dataset_name = ( "insitu-observations-near-surface-temperature-us-climate-reference-network" ) - source = "USCRN_HOURLY" + source = "uscrn_hourly" service_definition = get_service_definition(dataset_name) actual = get_aux_vars_from_service_definition(service_definition, source) print(actual) diff --git a/tests/test_config.py b/tests/test_config.py index 48502b0..8c11065 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,11 +7,11 @@ def test_config(test_config): "insitu-observations-near-surface-temperature-us-climate-reference-network" ) dataset_config = test_config.get_dataset(dataset_name) - lon_tile_size = dataset_config.get_tile_size("lon", "USCRN_MONTHLY", 2001) + lon_tile_size = dataset_config.get_tile_size("lon", "uscrn_monthly", 2001) assert lon_tile_size == 90 - lat_tile_size = dataset_config.get_tile_size("lat", "USCRN_HOURLY", 2010) + lat_tile_size = dataset_config.get_tile_size("lat", "uscrn_daily", 2010) assert lat_tile_size == 90 - lat_tile_size = dataset_config.get_tile_size("lat", "USCRN_SUBHOURLY", 2006) + lat_tile_size = dataset_config.get_tile_size("lat", "uscrn_subhourly", 2006) assert lat_tile_size == 30 - lat_tile_size = dataset_config.get_tile_size("lat", "USCRN_SUBHOURLY", 2015) + lat_tile_size = dataset_config.get_tile_size("lat", "uscrn_subhourly", 2015) assert lat_tile_size == 20 diff --git a/tests/test_http_api.py b/tests/test_http_api.py index 6da35dc..1aa17be 100644 --- a/tests/test_http_api.py +++ b/tests/test_http_api.py @@ -85,7 +85,7 @@ def test_get_dataset_auxiliary_variables_mapping(): dataset = ( "insitu-observations-near-surface-temperature-us-climate-reference-network" ) - actual = client.get(f"{dataset}/USCRN_DAILY/aux_variables_mapping").json() + actual = client.get(f"{dataset}/uscrn_daily/aux_variables_mapping").json() expected = { "accumulated_precipitation": [], "air_temperature": [ diff --git a/tests/upgrade_service_definition.py b/tests/upgrade_service_definition.py index 534603d..bcbf163 100644 --- a/tests/upgrade_service_definition.py +++ b/tests/upgrade_service_definition.py @@ -70,6 +70,8 @@ def main(old_path): new_values["dtype"] = "float32" new_values["long_name"] = rawname new_descriptions[cdm_name] = new_values + if "name_for_output" in new_values: + new_values.pop("name_for_output") # remap mandatory columns new_data["sources"][source]["mandatory_columns"] = [] for mcol in old_data["sources"][source]["mandatory_columns"]: @@ -89,7 +91,10 @@ def main(old_path): if __name__ == "__main__": - input_sd_files = "insitu-observations-gnss/service_definition.json" + input_sd_files = ( + "insitu-observations-near-surface-temperature-us-climate-" + "reference-network/service_definition.json" + ) for file in files("cdsobs").joinpath("data").glob(input_sd_files): # type: ignore print(file) main(file) diff --git a/tests/utils.py b/tests/utils.py index ed82f14..66c70d5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -26,16 +26,16 @@ def get_test_years(source: str) -> Tuple[int, int]: case "GRUAN": start_year = 2010 end_year = 2011 - case "USCRN_SUBHOURLY": + case "uscrn_subhourly": start_year = 2007 end_year = 2007 - case "USCRN_HOURLY": + case "uscrn_hourly": start_year = 2006 end_year = 2006 - case "USCRN_DAILY": + case "uscrn_daily": start_year = 2007 end_year = 2007 - case "USCRN_MONTHLY": + case "uscrn_monthly": start_year = 2006 end_year = 2006 case "IGS": From a291eb02eb7debb6cf0f409d3a56639f9849281c Mon Sep 17 00:00:00 2001 From: garciam Date: Wed, 24 Jul 2024 13:44:59 +0200 Subject: [PATCH 03/20] new service definition for USCRN --- .../service_definition.json | 56 +- .../service_definition.yml | 301 ++-- .../service_definition_old.yml | 1235 ----------------- tests/upgrade_service_definition.py | 2 +- 4 files changed, 130 insertions(+), 1464 deletions(-) delete mode 100644 cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json index 735c703..977092e 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.json @@ -50,12 +50,12 @@ "maximum_soil_temperature_quality_flag", "minimum_soil_temperature", "minimum_soil_temperature_quality_flag", - "solar_irradiance", - "solar_irradiance_quality_flag", - "maximum_solar_irradiance", - "maximum_solar_irradiance_quality_flag", - "minimum_solar_irradiance", - "minimum_solar_irradiance_quality_flag", + "downward_shortwave_irradiance_at_earth_surface", + "downward_shortwave_irradiance_at_earth_surface_quality_flag", + "downward_shortwave_irradiance_at_earth_surface_max", + "downward_shortwave_irradiance_at_earth_surface_max_quality_flag", + "downward_shortwave_irradiance_at_earth_surface_min", + "downward_shortwave_irradiance_at_earth_surface_min_quality_flag", "soil_moisture_5cm_from_earth_surface", "soil_moisture_10cm_from_earth_surface", "soil_moisture_20cm_from_earth_surface", @@ -244,9 +244,9 @@ }, "solar_radiation": { "units": "W m-2", - "name_for_output": "solar_irradiance", - "long_name": "solar_irradiance", - "description": "The quantity with standard name solar_irradiance, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. \"Irradiance\" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy.", + "name_for_output": "downward_shortwave_irradiance_at_earth_surface", + "long_name": "downward_shortwave_irradiance_at_earth_surface", + "description": "The quantity with standard name downward_shortwave_irradiance_at_earth_surface, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. \"Irradiance\" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy.", "quality_flag": "sr_flag" }, "surface_temperature": { @@ -291,9 +291,9 @@ "quality_flag": "wind_flag" }, "sr_flag": { - "name_for_output": "solar_irradiance_quality_flag", - "long_name": "solar_irradiance_quality_flag", - "description": "Quality indicator variable 'solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + "name_for_output": "downward_shortwave_irradiance_at_earth_surface_quality_flag", + "long_name": "downward_shortwave_irradiance_at_earth_surface_quality_flag", + "description": "Quality indicator variable 'downward_shortwave_irradiance_at_earth_surface'. 0 denotes good data and 3 denotes erroneous data." }, "st_flag": { "name_for_output": "soil_temperature_processing_level_quality_flag", @@ -583,39 +583,39 @@ }, "solarad": { "units": "W m-2", - "name_for_output": "solar_irradiance", - "long_name": "solar_irradiance", - "description": "The quantity with standard name solar_irradiance, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. \"Irradiance\" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy.", + "name_for_output": "downward_shortwave_irradiance_at_earth_surface", + "long_name": "downward_shortwave_irradiance_at_earth_surface", + "description": "The quantity with standard name downward_shortwave_irradiance_at_earth_surface, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. \"Irradiance\" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy.", "quality_flag": "solarad_flag" }, "solarad_flag": { - "name_for_output": "solar_irradiance_quality_flag", - "long_name": "solar_irradiance_quality_flag", - "description": "Quality indicator variable 'solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + "name_for_output": "downward_shortwave_irradiance_at_earth_surface_quality_flag", + "long_name": "downward_shortwave_irradiance_at_earth_surface_quality_flag", + "description": "Quality indicator variable 'downward_shortwave_irradiance_at_earth_surface'. 0 denotes good data and 3 denotes erroneous data." }, "solarad_max": { "units": "W m-2", - "name_for_output": "maximum_solar_irradiance", - "long_name": "maximum_solar_irradiance", + "name_for_output": "downward_shortwave_irradiance_at_earth_surface_max", + "long_name": "downward_shortwave_irradiance_at_earth_surface_max", "description": "Maximum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI units at the Earth surface over specified period.", "quality_flag": "solarad_max_flag" }, "solarad_max_flag": { - "name_for_output": "maximum_solar_irradiance_quality_flag", - "long_name": "maximum_solar_irradiance_quality_flag", - "description": "Quality indicator for the variable 'maximum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + "name_for_output": "downward_shortwave_irradiance_at_earth_surface_max_quality_flag", + "long_name": "downward_shortwave_irradiance_at_earth_surface_max_quality_flag", + "description": "Quality indicator for the variable 'downward_shortwave_irradiance_at_earth_surface_max'. 0 denotes good data and 3 denotes erroneous data." }, "solarad_min": { "units": "W m-2", - "name_for_output": "minimum_solar_irradiance", - "long_name": "minimum_downward_shortwave_irradiance_at_earth_surfac", + "name_for_output": "downward_shortwave_irradiance_at_earth_surface_min", + "long_name": "downward_shortwave_irradiance_at_earth_surface_min", "description": "Minimum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI unitsat the Earth surface.", "quality_flag": "solarad_min_flag" }, "solarad_min_flag": { - "name_for_output": "minimum_solar_irradiance_quality_flag", - "long_name": "minimum_solar_irradiance_quality_flag", - "description": "Quality indicator for variable 'minimum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data." + "name_for_output": "downward_shortwave_irradiance_at_earth_surface_min_quality_flag", + "long_name": "downward_shortwave_irradiance_at_earth_surface_min_quality_flag", + "description": "Quality indicator for variable 'downward_shortwave_irradiance_at_earth_surface_min'. 0 denotes good data and 3 denotes erroneous data." }, "sur_temp": { "units": "K", diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml index 3f69312..eafd83e 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml @@ -3,7 +3,7 @@ global_attributes: licence_list: 20180314_Copernicus_License_V1.1 responsible_organisation: ECMWF out_columns_order: -- primary_station_id +- station_name - alternative_name - report_timestamp - report_id @@ -21,25 +21,25 @@ out_columns_order: - air_temperature_negative_total_uncertainty - air_temperature_random_uncertainty - air_temperature_quasisystematic_uncertainty -- air_temperature_max -- air_temperature_max_negative_total_uncertainty -- air_temperature_max_positive_total_uncertainty -- air_temperature_min -- air_temperature_min_negative_total_uncertainty -- air_temperature_min_positive_total_uncertainty -- air_temperature_mean -- air_temperature_mean_negative_total_uncertainty -- air_temperature_mean_positive_total_uncertainty +- maximum_air_temperature +- maximum_air_temperature_negative_total_uncertainty +- maximum_air_temperature_positive_total_uncertainty +- minimum_air_temperature +- minimum_air_temperature_negative_total_uncertainty +- minimum_air_temperature_positive_total_uncertainty +- mean_air_temperature +- mean_air_temperature_negative_total_uncertainty +- mean_air_temperature_positive_total_uncertainty - relative_humidity - relative_humidity_quality_flag -- relative_humidity_max -- relative_humidity_min +- maximum_relative_humidity +- minimum_relative_humidity - soil_temperature - soil_temperature_quality_flag -- soil_temperature_max -- soil_temperature_max_quality_flag -- soil_temperature_min -- soil_temperature_min_quality_flag +- maximum_soil_temperature +- maximum_soil_temperature_quality_flag +- minimum_soil_temperature +- minimum_soil_temperature_quality_flag - solar_irradiance - solar_irradiance_quality_flag - maximum_solar_irradiance @@ -63,20 +63,14 @@ out_columns_order: - wetness - wetness_quality_flag - accumulated_precipitation -- daily_global_solar_radiation_rates -- monthly_global_solar_radiation_rates +- daily_global_solar_radiation +- monthly_global_solar_radiation products_hierarchy: - variables - flag - processing_level - positive_total_uncertainty - negative_total_uncertainty -- max_positive_total_uncertainty -- max_negative_total_uncertainty -- min_positive_total_uncertainty -- min_negative_total_uncertainty -- mean_positive_total_uncertainty -- mean_negative_total_uncertainty - random_uncertainty - positive_random_uncertainty - negative_random_uncertainty @@ -134,117 +128,14 @@ sources: tminerr_m: air_temperature_min_negative_total_uncertainty tminerr_p: air_temperature_min_positive_total_uncertainty wbanno: primary_station_id + wind_1_5: wind_speed_2_meters_from_earth_surface + wind_flag: wind_speed_2_meters_from_earth_surface_quality_flag unit_changes: accumulated_precipitation: names: mm: mm offset: 0 scale: 1 - air_temperature: - names: - K: K - offset: 0 - scale: 1 - daily_global_solar_radiation: - names: - MJ m-2: MJ m-2 - offset: 0 - scale: 1 - maximum_air_temperature: - names: - K: K - offset: 0 - scale: 1 - maximum_relative_humidity: - names: - '%': '%' - offset: 0 - scale: 1 - maximum_soil_temperature: - names: - K: K - offset: 0 - scale: 1 - mean_air_temperature: - names: - K: K - offset: 0 - scale: 1 - minimum_air_temperature: - names: - K: K - offset: 0 - scale: 1 - minimum_relative_humidity: - names: - '%': '%' - offset: 0 - scale: 1 - minimum_soil_temperature: - names: - K: K - offset: 0 - scale: 1 - relative_humidity: - names: - '%': '%' - offset: 0 - scale: 1 - soil_moisture_100cm_from_earth_surface: - names: - m3 m-3: m3 m-3 - offset: 0 - scale: 1 - soil_moisture_10cm_from_earth_surface: - names: - m3 m-3: m3 m-3 - offset: 0 - scale: 1 - soil_moisture_20cm_from_earth_surface: - names: - m3 m-3: m3 m-3 - offset: 0 - scale: 1 - soil_moisture_50cm_from_earth_surface: - names: - m3 m-3: m3 m-3 - offset: 0 - scale: 1 - soil_moisture_5cm_from_earth_surface: - names: - m3 m-3: m3 m-3 - offset: 0 - scale: 1 - soil_temperature: - names: - K: K - offset: 0 - scale: 1 - soil_temperature_100cm_from_earth_surface: - names: - K: K - offset: 0 - scale: 1 - soil_temperature_10cm_from_earth_surface: - names: - K: K - offset: 0 - scale: 1 - soil_temperature_20cm_from_earth_surface: - names: - K: K - offset: 0 - scale: 1 - soil_temperature_50cm_from_earth_surface: - names: - K: K - offset: 0 - scale: 1 - soil_temperature_5cm_from_earth_surface: - names: - K: K - offset: 0 - scale: 1 data_table: unc_daily descriptions: accumulated_precipitation: @@ -326,24 +217,24 @@ sources: dtype: float32 long_name: longitude units: degree_east - maximum_air_temperature: + daily_maximum_air_temperature: description: "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_daily_max negative_total_uncertainty: tmaxerr_m positive_total_uncertainty: tmaxerr_p units: K - maximum_air_temperature_negative_total_uncertainty: + daily_maximum_air_temperature_negative_total_uncertainty: description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tmaxerr_m units: K - maximum_air_temperature_positive_total_uncertainty: + daily_maximum_air_temperature_positive_total_uncertainty: description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tmaxerr_p units: K - maximum_relative_humidity: + daily_maximum_relative_humidity: description: From measurements of "Relative humidity" (described in this list), this parameter provides the highest value measured during the aggregation period. dtype: float32 long_name: rh_daily_max @@ -353,7 +244,7 @@ sources: dtype: float32 long_name: sur_temp_daily_max units: K - mean_air_temperature: + daily_mean_air_temperature: description: "This parameter is the average of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_daily_mean @@ -370,24 +261,24 @@ sources: dtype: float32 long_name: tmeanerr_p units: K - minimum_air_temperature: + daily_minimum_air_temperature: description: "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_daily_min negative_total_uncertainty: tminerr_m positive_total_uncertainty: tminerr_p units: K - minimum_air_temperature_negative_total_uncertainty: + daily_minimum_air_temperature_negative_total_uncertainty: description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tminerr_m units: K - minimum_air_temperature_positive_total_uncertainty: + daily_minimum_air_temperature_positive_total_uncertainty: description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tminerr_p units: K - minimum_relative_humidity: + daily_minimum_relative_humidity: description: From measurements of "Relative humidity" (described in this list), this parameter provides the lowest value measured during the aggregation period. dtype: float32 long_name: rh_daily_min @@ -470,7 +361,7 @@ sources: description: 'This parameter indicates the level of processing applied to the soil temperature measurement: raw data (''R''), corrected data (''C''). A letter ''U'' indicates that this information is unknown.' dtype: float32 long_name: sur_temp_daily_type - primary_station_id: + station_name: description: This is the station identification code. dtype: object long_name: wbanno @@ -656,6 +547,36 @@ sources: description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object long_name: sitename + solar_irradiance: + description: The quantity with standard name downward_shortwave_irradiance_at_earth_surface, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. "Irradiance" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy. + dtype: float32 + long_name: solarad + quality_flag: solarad_flag + units: W m-2 + maximum_solar_irradiance: + description: Maximum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI units at the Earth surface over specified period. + dtype: float32 + long_name: solarad_max + quality_flag: solarad_max_flag + units: W m-2 + maximum_solar_irradiance_quality_flag: + description: Quality indicator for the variable 'downward_shortwave_irradiance_at_earth_surface_max'. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: solarad_max_flag + minimum_solar_irradiance: + description: Minimum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI unitsat the Earth surface. + dtype: float32 + long_name: solarad_min + quality_flag: solarad_min_flag + units: W m-2 + minimum_solar_irradiance_quality_flag: + description: Quality indicator for variable 'downward_shortwave_irradiance_at_earth_surface_min'. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: solarad_min_flag + solar_irradiance_quality_flag: + description: Quality indicator variable 'downward_shortwave_irradiance_at_earth_surface'. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: solarad_flag latitude: description: Latitude of the measurement station, -90 to 90. dtype: float32 @@ -670,80 +591,60 @@ sources: dtype: float32 long_name: longitude units: degree_east - maximum_air_temperature: + daily_maximum_air_temperature: description: "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_max max_negative_total_uncertainty: tmaxerr_m max_positive_total_uncertainty: tmaxerr_p units: K - maximum_air_temperature_negative_total_uncertainty: + daily_maximum_air_temperature_negative_total_uncertainty: description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tmaxerr_m units: K - maximum_air_temperature_positive_total_uncertainty: + daily_maximum_air_temperature_positive_total_uncertainty: description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tmaxerr_p units: K - maximum_soil_temperature: + hourly_maximum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the highest value measured during the aggregation period. dtype: float32 long_name: sur_temp_max quality_flag: sur_temp_max_flag units: K - maximum_soil_temperature_quality_flag: + hourly_maximum_soil_temperature_quality_flag: description: This parameter indicates if the "Maximum soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). denotes good data and 3 denotes erroneous data. dtype: float32 long_name: sur_temp_max_flag - maximum_solar_irradiance: - description: Maximum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI units at the Earth surface over specified period. - dtype: float32 - long_name: solarad_max - quality_flag: solarad_max_flag - units: W m-2 - maximum_solar_irradiance_quality_flag: - description: Quality indicator for the variable 'maximum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: solarad_max_flag - minimum_air_temperature: + daily_minimum_air_temperature: description: "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_min min_negative_total_uncertainty: tminerr_m min_positive_total_uncertainty: tminerr_p units: K - minimum_air_temperature_negative_total_uncertainty: + daily_minimum_air_temperature_negative_total_uncertainty: description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tminerr_m units: K - minimum_air_temperature_positive_total_uncertainty: + daily_minimum_air_temperature_positive_total_uncertainty: description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tminerr_p units: K - minimum_soil_temperature: + hourly_minimum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the lowest value measured during the aggregation period. dtype: float32 long_name: sur_temp_min quality_flag: sur_temp_min_flag units: K - minimum_soil_temperature_quality_flag: + hourly_minimum_soil_temperature_quality_flag: description: This parameter indicates if the "Minimum soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). dtype: float32 long_name: sur_temp_min_flag - minimum_solar_irradiance: - description: Minimum power per unit area (surface power density) received from the Sun in the form of electromagnetic radiation in the wavelength range of the measuring instrument. Solar irradiance is measured in watts per square metre (W/m2) in SI unitsat the Earth surface. - dtype: float32 - long_name: solarad_min - quality_flag: solarad_min_flag - units: W m-2 - minimum_solar_irradiance_quality_flag: - description: Quality indicator for variable 'minimum_solar_irradiance'. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: solarad_min_flag relative_humidity: description: This parameter is the ratio of the amount of atmospheric moisture present in the air relative to the amount that would be present if the air were saturated with respect to water or ice. This ratio is multiplied by 100 to provide numbers in percent (%). The measurements are taken at a height of 2 metres above the ground surface. The instrument is a capacitive thin-film device. It measures the flow of electricity across two electrodes separated by a polymer film. The capacitance of the film is converted to relative humidity by a calibration equation. The values are averaged over 5-minute periods, obtained by selecting "sub-hourly" aggregation period. Other aggregation periods ("hourly", "daily") are also available, in which case the results are based on further averages of the 5-minute data. A quality flag is also available for this variable. dtype: float32 @@ -827,28 +728,18 @@ sources: description: This parameter indicates if the "Soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). dtype: float32 long_name: sur_temp_flag - solar_irradiance: - description: The quantity with standard name solar_irradiance, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. "Irradiance" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy. - dtype: float32 - long_name: solarad - quality_flag: solarad_flag - units: W m-2 - solar_irradiance_quality_flag: - description: Quality indicator variable 'solar_irradiance'. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: solarad_flag - primary_station_id: + station_name: description: This is the station identification code. - dtype: object + dtype: float32 long_name: wbanno mandatory_columns: - alternative_name - - primary_station_id - - longitude|station_configuration - - latitude|station_configuration + - station_name + - longitude + - latitude - report_timestamp order_by: - - primary_station_id + - station_name - report_timestamp products: - columns: @@ -915,7 +806,7 @@ sources: - sur_temp_max_flag - sur_temp_min_flag - rh_hr_avg_flag - group_name: flag + group_name: quality_flag - columns: - sur_temp_type group_name: processing_level @@ -943,7 +834,7 @@ sources: sys_p: air_temperature_positive_systematic_uncertainty t: air_temperature t_monthly_max: daily_maximum_air_temperature - t_monthly_mean: air_temperature + t_monthly_mean: daily_mean_air_temperature t_monthly_min: daily_minimum_air_temperature terr_m: air_temperature_negative_total_uncertainty terr_p: air_temperature_positive_total_uncertainty @@ -1036,51 +927,51 @@ sources: dtype: float32 long_name: longitude units: degree_east - maximum_air_temperature: + daily_maximum_air_temperature: description: "This parameter is the highest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_monthly_max negative_total_uncertainty: tmaxerr_m positive_total_uncertainty: tmaxerr_p units: K - maximum_air_temperature_negative_total_uncertainty: + daily_maximum_air_temperature_total_uncertainty: description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tmaxerr_m units: K - maximum_air_temperature_positive_total_uncertainty: + daily_maximum_air_temperature_positive_total_uncertainty: description: "For \"Maximum air temperature\" (described in this list), this parameter is a partial estimate of errors (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tmaxerr_p units: K - maximum_soil_temperature: + monthly_maximum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the highest value measured during the aggregation period. dtype: float32 long_name: sur_temp_monthly_max units: K - mean_air_temperature: + daily_mean_air_temperature: description: "This parameter is the average of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_monthly_mean units: K - minimum_air_temperature: + daily_minimum_air_temperature: description: "This parameter is the lowest of all sub-hourly (5-minute) values of air temperature recorded at the station during the aggregation period selected by the user. It is based on measurements made at the station, typically at about 1.5 metres above the ground surface. This parameter has units of Kelvin (K). Temperature measured in Kelvin can be converted to degrees Celsius (°C) by subtracting 273.15. Two error estimates are provided in addition, to help users take into account uncertainties." dtype: float32 long_name: t_monthly_min negative_total_uncertainty: tminerr_m positive_total_uncertainty: tminerr_p units: K - minimum_air_temperature_negative_total_uncertainty: + daily_minimum_air_temperature_negative_total_uncertainty: description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Negative total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tminerr_m units: K - minimum_air_temperature_positive_total_uncertainty: + daily_minimum_air_temperature_positive_total_uncertainty: description: "For \"Minimum air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive total uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." dtype: float32 long_name: tminerr_p units: K - minimum_soil_temperature: + monthly_minimum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the lowest value measured during the aggregation period. dtype: float32 long_name: sur_temp_monthly_min @@ -1204,16 +1095,16 @@ sources: scale: 1 data_table: unc_subhourly descriptions: - 2m_wind_speed: + wind_speed_2_meters_from_earth_surface: description: This parameter is the horizontal velocity of the air near the surface. It is measured using a 3-cup anemometer placed at the same height at the air temperature shield intake. The exact measurement height is 1.5 metres above the ground surface. dtype: float32 - long_name: wind_1_5 - quality_flag: wind_flag + long_name: wind_speed_2_meters_from_earth_surface + quality_flag: wind_speed_2_meters_from_earth_surface_quality_flag units: m s-1 2m_wind_speed_quality_flag: description: This parameter indicates if the "2m wind speed" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (non-zero value). dtype: float32 - long_name: wind_flag + long_name: wind_speed_2_meters_from_earth_surface_quality_flag accumulated_precipitation: description: This parameter is the accumulated precipitation that was measured as having reached the Earth's surface at the station. It includes all forms of precipitation (rain and snow). This parameter is accumulated over the aggregation period selected by the user. The units of this parameter are depth in millimetre of water equivalent. Care should be taken when comparing such observations with model or reanalysis parameters, because observations are by definition local to a particular point in space, rather than representing averages over a large geographical area. dtype: float32 @@ -1258,6 +1149,16 @@ sources: description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object long_name: sitename + downward_shortwave_irradiance_at_earth_surface: + description: The quantity with standard name downward_shortwave_irradiance_at_earth_surface, often called Total Solar Irradiance (TSI), is the radiation from the sun integrated over the whole electromagnetic spectrum and over the entire solar disk. The quantity applies outside the atmosphere, by default at a distance of one astronomical unit from the sun, but a coordinate or scalar coordinate variable of distance_from_sun can be used to specify a value other than the default. "Irradiance" means the power per unit area (called radiative flux in other standard names), the area being normal to the direction of flow of the radiant energy. + dtype: float32 + long_name: solar_radiation + quality_flag: sr_flag + units: W m-2 + downward_shortwave_irradiance_at_earth_surface_quality_flag: + description: Quality indicator variable 'downward_shortwave_irradiance_at_earth_surface'. 0 denotes good data and 3 denotes erroneous data. + dtype: float32 + long_name: sr_flag latitude: description: Latitude of the measurement station, -90 to 90. dtype: float32 diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml deleted file mode 100644 index f154064..0000000 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition_old.yml +++ /dev/null @@ -1,1235 +0,0 @@ -global_attributes: - contactemail: https://support.ecmwf.int - licence_list: 20180314_Copernicus_License_V1.1 - responsible_organisation: ECMWF -out_columns_order: -- primary_station_id -- alternative_name -- report_timestamp -- report_id -- logbook_version -- longitude -- latitude -- air_temperature -- air_temperature_positive_random_uncertainty -- air_temperature_positive_systematic_uncertainty -- air_temperature_positive_quasisystematic_uncertainty -- air_temperature_positive_total_uncertainty -- air_temperature_negative_random_uncertainty -- air_temperature_negative_systematic_uncertainty -- air_temperature_negative_quasisystematic_uncertainty -- air_temperature_negative_total_uncertainty -- air_temperature_random_uncertainty -- air_temperature_quasisystematic_uncertainty -- air_temperature_max -- air_temperature_max_negative_total_uncertainty -- air_temperature_max_positive_total_uncertainty -- air_temperature_min -- air_temperature_min_negative_total_uncertainty -- air_temperature_min_positive_total_uncertainty -- air_temperature_mean -- air_temperature_mean_negative_total_uncertainty -- air_temperature_mean_positive_total_uncertainty -- relative_humidity -- relative_humidity_quality_flag -- relative_humidity_max -- relative_humidity_min -- soil_temperature -- soil_temperature_quality_flag -- soil_temperature_max -- soil_temperature_max_quality_flag -- soil_temperature_min -- soil_temperature_min_quality_flag -- downward_shortwave_irradiance_at_earth_surface -- downward_shortwave_irradiance_at_earth_surface_quality_flag -- downward_shortwave_irradiance_at_earth_surface_max -- downward_shortwave_irradiance_at_earth_surface_max_quality_flag -- downward_shortwave_irradiance_at_earth_surface_min -- downward_shortwave_irradiance_at_earth_surface_min_quality_flag -- soil_moisture_5cm_from_earth_surface -- soil_moisture_10cm_from_earth_surface -- soil_moisture_20cm_from_earth_surface -- soil_moisture_50cm_from_earth_surface -- soil_moisture_100cm_from_earth_surface -- soil_temperature_5cm_from_earth_surface -- soil_temperature_10cm_from_earth_surface -- soil_temperature_20cm_from_earth_surface -- soil_temperature_50cm_from_earth_surface -- soil_temperature_100cm_from_earth_surface -- soil_temperature_processing_level -- soil_temperature_processing_level_quality_flag -- wind_speed_2_meters_from_earth_surface -- wind_speed_2_meters_from_earth_surface_quality_flag -- wetness -- wetness_quality_flag -- accumulated_precipitation -- daily_global_solar_radiation_rates -- monthly_global_solar_radiation_rates -products_hierarchy: -- variables -- flag -- processing_level -- positive_total_uncertainty -- negative_total_uncertainty -- max_positive_total_uncertainty -- max_negative_total_uncertainty -- min_positive_total_uncertainty -- min_negative_total_uncertainty -- mean_positive_total_uncertainty -- mean_negative_total_uncertainty -- random_uncertainty -- positive_random_uncertainty -- negative_random_uncertainty -- positive_systematic_uncertainty -- negative_systematic_uncertainty -- quasisystematic_uncertainty -- positive_quasisystematic_uncertainty -- negative_quasisystematic_uncertainty -sources: - USCRN_DAILY: - cdm_mapping: - melt_columns: true - rename: - crx_vn: logbook_version - date_of_observation: report_timestamp - id: report_id - latitude: latitude|station_configuration - longitude: longitude|station_configuration - p_daily_calc: accumulated_precipitation - quasisys_m: air_temperature_negative_quasisystematic_uncertainty - quasisys_p: air_temperature_positive_quasisystematic_uncertainty - random_m: air_temperature_negative_random_uncertainty - random_p: air_temperature_positive_random_uncertainty - rh_daily_avg: relative_humidity - rh_daily_max: daily_maximum_relative_humidity - rh_daily_min: daily_minimum_relative_humidity - sitename: alternative_name - soil_moisture_100_daily: soil_moisture_100cm_from_earth_surface - soil_moisture_10_daily: soil_moisture_10cm_from_earth_surface - soil_moisture_20_daily: soil_moisture_20cm_from_earth_surface - soil_moisture_50_daily: soil_moisture_50cm_from_earth_surface - soil_moisture_5_daily: soil_moisture_5cm_from_earth_surface - soil_temp_100_daily: soil_temperature_100cm_from_earth_surface - soil_temp_10_daily: soil_temperature_10cm_from_earth_surface - soil_temp_20_daily: soil_temperature_20cm_from_earth_surface - soil_temp_50_daily: soil_temperature_50cm_from_earth_surface - soil_temp_5_daily: soil_temperature_5cm_from_earth_surface - sur_temp_daily_avg: soil_temperature - sur_temp_daily_type: soil_temperature_processing_level - sys_m: air_temperature_negative_systematic_uncertainty - sys_p: air_temperature_positive_systematic_uncertainty - t: air_temperature - t_daily_max: daily_maximum_air_temperature - t_daily_mean: daily_mean_air_temperature - t_daily_min: daily_minimum_air_temperature - terr_m: air_temperature_negative_total_uncertainty - terr_p: air_temperature_positive_total_uncertainty - tmaxerr_m: air_temperature_max_negative_total_uncertainty - tmaxerr_p: air_temperature_max_positive_total_uncertainty - tmeanerr_m: air_temperature_mean_negative_total_uncertainty - tmeanerr_p: air_temperature_mean_positive_total_uncertainty - tminerr_m: air_temperature_min_negative_total_uncertainty - tminerr_p: air_temperature_min_positive_total_uncertainty - wbanno: primary_station_id - unit_changes: - accumulated_precipitation: - names: - mm: mm - offset: 0 - scale: 1 - data_table: unc_daily - descriptions: - accumulated_precipitation: - description: Total amount of precipitation - dtype: float32 - long_name: accumulated_precipitation - units: mm - air_temperature: - description: 'Mean air temperature calculated using the typical historical approach: (T_DAILY_MAX + T_DAILY_MIN) / 2' - dtype: float32 - long_name: mean_air_temperature - negative_total_uncertainty: tmeanerr_m - positive_total_uncertainty: tmeanerr_p - units: K - air_temperature_max_negative_total_uncertainty: - description: negative uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_negative_total_uncertainty - units: K - air_temperature_max_positive_total_uncertainty: - description: positive uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_positive_total_uncertainty - units: K - air_temperature_mean_negative_total_uncertainty: - description: tmeanerr_m, tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (negative per using the Hourly data) - dtype: float32 - long_name: mean_air_temperature_negative_total_uncertainty - units: K - air_temperature_mean_positive_total_uncertainty: - description: tmean is (tmax+tmin)/2 and the uncertianty is the average of the uncertaities of tmax and tmin (positive part using the Hourly data) - dtype: float32 - long_name: mean_air_temperature_positive_total_uncertainty - units: K - air_temperature_min_negative_total_uncertainty: - description: negative uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_negative_total_uncertainty - units: K - air_temperature_min_positive_total_uncertainty: - description: positive uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_positive_total_uncertainty - units: K - air_temperature_negative_quasisystematic_uncertainty: - description: -ve/-ve Quasi-systematic - dtype: float32 - long_name: air_temperature_negative_quasisystematic_uncertainty - units: K - air_temperature_negative_random_uncertainty: - description: -ve Random uncertainty - dtype: float32 - long_name: air_temperature_negative_random_uncertainty - units: K - air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty - units: K - air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature - dtype: float32 - long_name: air_temperature_negative_total_uncertainty - units: K - air_temperature_positive_quasisystematic_uncertainty: - description: +ve Quasi-systematic uncertainty - dtype: float32 - long_name: air_temperature_positive_quasisystematic_uncertainty - units: K - air_temperature_positive_random_uncertainty: - description: +ve Random uncertainty - dtype: float32 - long_name: air_temperature_positive_random_uncertainty - units: K - air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty - units: K - air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature - dtype: float32 - long_name: air_temperature_positive_total_uncertainty - units: K - alternative_name: - description: Alternative name for station - dtype: object - long_name: sitename - daily_maximum_air_temperature: - description: Maximum air temperature - dtype: float32 - long_name: air_temperature_max - negative_total_uncertainty: tmaxerr_m - positive_total_uncertainty: tmaxerr_p - units: K - daily_minimum_air_temperature: - description: Minimum air temperature - dtype: float32 - long_name: minimum_air_temperature - negative_total_uncertainty: tminerr_m - positive_total_uncertainty: tminerr_p - units: K - location_latitude: - description: Latitude of the station (deg. North) - dtype: float32 - long_name: latitude - units: degree_north - location_longitude: - description: Longitude of the station (deg. East) - dtype: float32 - long_name: longitude - units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn - relative_humidity: - description: Relative humidity is computed from 5-minute values in almost all cases. All USCRN stations now report 5-minute averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. - dtype: float32 - long_name: average_relative_humidity - units: '%' - daily_maximum_relative_humidity: - description: Maximum relative humidity - dtype: float32 - long_name: maximum_relative_humidity - units: '%' - daily_minimum_relative_humidity: - description: Minimum relative humidity - dtype: float32 - long_name: minimum_relative_humidity - units: '%' - report_id: - description: Identifier in the USCRN meta-database - long_name: id - report_timestamp: - description: observation date time UTC - dtype: datetime64[ns] - long_name: date_of_observation - soil_moisture_100cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 100 cm below the surface. - dtype: float32 - long_name: soil_moisture_100cm_from_earth_surface - units: m3 m-3 - soil_moisture_10cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 10 cm below the surface. - dtype: float32 - long_name: soil_moisture_10cm_from_earth_surface - units: m3 m-3 - soil_moisture_20cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 20 cm below the surface. - dtype: float32 - long_name: soil_moisture_20cm_from_earth_surface - units: m3 m-3 - soil_moisture_50cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 50 cm below the surface. - dtype: float32 - long_name: soil_moisture_50cm_from_earth_surface - units: m3 m-3 - soil_moisture_5cm_from_earth_surface: - description: Average soil moisture, in fractional volumetric water content (m^3 m^-3), at 5 cm below the surface. - dtype: float32 - long_name: soil_moisture_5cm_from_earth_surface - units: m3 m-3 - soil_temperature: - description: Average infrared surface temperature - dtype: float32 - long_name: average_soil_temperature - processing_level: sur_temp_daily_type - units: K - soil_temperature_100cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 100 cm below the surface. - dtype: float32 - long_name: soil_temperature_100cm_from_earth_surface - units: K - soil_temperature_10cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 10 cm below the surface. - dtype: float32 - long_name: soil_temperature_10cm_from_earth_surface - units: K - soil_temperature_20cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 20 cm below the surface. - dtype: float32 - long_name: soil_temperature_20cm_from_earth_surface - units: K - soil_temperature_50cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 50 cm below the surface. - dtype: float32 - long_name: soil_temperature_50cm_from_earth_surface - units: K - soil_temperature_5cm_from_earth_surface: - description: Average soil temperature, in degrees C, at 5 cm below the surface. - dtype: float32 - long_name: soil_temperature_5cm_from_earth_surface - units: K - soil_temperature_processing_level: - description: Type of infrared surface temperature measurement. - dtype: float32 - long_name: soil_temperature_processing_level - primary_station_id: - description: Station identification code - dtype: object - long_name: wbanno - mandatory_columns: - - alternative_name - - primary_station_id - - longitude|station_configuration - - latitude|station_configuration - - report_timestamp - order_by: - - primary_station_id - - report_timestamp - products: - - columns: - - t_daily_max - - t_daily_min - - t_daily_mean - - p_daily_calc - - sur_temp_daily_avg - - rh_daily_max - - rh_daily_min - - rh_daily_avg - - soil_moisture_5_daily - - soil_moisture_10_daily - - soil_moisture_20_daily - - soil_moisture_50_daily - - soil_moisture_100_daily - - soil_temp_5_daily - - soil_temp_10_daily - - soil_temp_20_daily - - soil_temp_50_daily - - soil_temp_100_daily - group_name: variables - - columns: - - random_p - group_name: positive_random_uncertainty - - columns: - - random_m - group_name: negative_random_uncertainty - - columns: - - terr_p - - tmaxerr_p - - tminerr_p - - tmeanerr_p - group_name: positive_total_uncertainty - - columns: - - terr_m - - tmaxerr_m - - tminerr_m - - tmeanerr_m - group_name: negative_total_uncertainty - - columns: - - quasisys_p - group_name: positive_quasisystematic_uncertainty - - columns: - - quasisys_m - group_name: negative_quasisystematic_uncertainty - - columns: - - sys_p - group_name: positive_systematic_uncertainty - - columns: - - sys_m - group_name: negative_systematic_uncertainty - - columns: - - sur_temp_daily_type - group_name: processing_level - USCRN_HOURLY: - cdm_mapping: - melt_columns: true - rename: - crx_vn: logbook_version - date_of_observation: report_timestamp - id: report_id - latitude: latitude|station_configuration - longitude: longitude|station_configuration - p_calc: accumulated_precipitation - quasisys_m: air_temperature_negative_quasisystematic_uncertainty - quasisys_p: air_temperature_positive_quasisystematic_uncertainty - random_pm: air_temperature_random_uncertainty - rh_hr_avg: relative_humidity - rh_hr_avg_flag: relative_humidity_quality_flag - sitename: alternative_name - soil_moisture_10: soil_moisture_10cm_from_earth_surface - soil_moisture_100: soil_moisture_100cm_from_earth_surface - soil_moisture_20: soil_moisture_20cm_from_earth_surface - soil_moisture_5: soil_moisture_5cm_from_earth_surface - soil_moisture_50: soil_moisture_50cm_from_earth_surface - soil_temp_10: soil_temperature_10cm_from_earth_surface - soil_temp_100: soil_temperature_100cm_from_earth_surface - soil_temp_20: soil_temperature_20cm_from_earth_surface - soil_temp_5: soil_temperature_5cm_from_earth_surface - soil_temp_50: soil_temperature_50cm_from_earth_surface - solarad: downward_shortwave_irradiance_at_earth_surface - solarad_flag: downward_shortwave_irradiance_at_earth_surface_quality_flag - solarad_max: hourly_maximum_downward_shortwave_irradiance_at_earth_surface - solarad_max_flag: hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag - solarad_min: hourly_minimum_downward_shortwave_irradiance_at_earth_surface - solarad_min_flag: hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag - sur_temp: soil_temperature - sur_temp_flag: soil_temperature_quality_flag - sur_temp_max: hourly_maximum_soil_temperature - sur_temp_max_flag: hourly_maximum_soil_temperature_flag - sur_temp_min: hourly_minimum_soil_temperature - sur_temp_min_flag: hourly_minimum_soil_temperature_quality_flag - sur_temp_type: soil_temperature_processing_level - sys_m: air_temperature_negative_systematic_uncertainty - sys_p: air_temperature_positive_systematic_uncertainty - t: air_temperature - t_max: daily_maximum_air_temperature - t_min: daily_minimum_air_temperature - terr_m: air_temperature_negative_total_uncertainty - terr_p: air_temperature_positive_total_uncertainty - tmaxerr_m: air_temperature_max_negative_total_uncertainty - tmaxerr_p: air_temperature_max_positive_total_uncertainty - tminerr_m: air_temperature_min_negative_total_uncertainty - tminerr_p: air_temperature_min_positive_total_uncertainty - wbanno: primary_station_id - unit_changes: - accumulated_precipitation: - names: - mm: mm - offset: 0 - scale: 1 - data_table: unc_hourly - descriptions: - accumulated_precipitation: - description: Total amount of precipitation, in mm, recorded during the hour - dtype: float32 - long_name: accumulated_precipitation - units: mm - air_temperature: - description: Near surface temperature - dtype: float32 - long_name: average_air_temperature - max_negative_total_uncertainty: tmaxerr_m - max_positive_total_uncertainty: tmaxerr_p - min_negative_total_uncertainty: tminerr_m - min_positive_total_uncertainty: tminerr_p - negative_quasisystematic_uncertainty: quasisys_m - negative_systematic_uncertainty: sys_m - negative_total_uncertainty: terr_m - positive_quasisystematic_uncertainty: quasisys_p - positive_systematic_uncertainty: sys_p - positive_total_uncertainty: terr_p - random_uncertainty: random_pm - units: K - air_temperature_max_negative_total_uncertainty: - description: negative uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_negative_total_uncertainty - units: K - air_temperature_max_positive_total_uncertainty: - description: positive uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_positive_total_uncertainty - units: K - air_temperature_min_negative_total_uncertainty: - description: negative uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_negative_total_uncertainty - units: K - air_temperature_min_positive_total_uncertainty: - description: positive uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_positive_total_uncertainty - units: K - air_temperature_negative_quasisystematic_uncertainty: - description: -ve/-ve Quasi-systematic - dtype: float32 - long_name: air_temperature_negative_quasisystematic_uncertainty - units: K - air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty - units: K - air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature - dtype: float32 - long_name: air_temperature_negative_total_uncertainty - units: K - air_temperature_positive_quasisystematic_uncertainty: - description: +ve Quasi-systematic uncertainty - dtype: float32 - long_name: air_temperature_positive_quasisystematic_uncertainty - units: K - air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty - units: K - air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature - dtype: float32 - long_name: air_temperature_positive_total_uncertainty - units: K - air_temperature_random_uncertainty: - description: +ve/-ve Random - dtype: float32 - long_name: air_temperature_random_uncertainty - units: K - alternative_name: - description: Alternative name for station - dtype: object - long_name: sitename - daily_maximum_air_temperature: - description: Maximum air temperature during the hour - dtype: float32 - long_name: air_temperature_max - units: K - daily_minimum_air_temperature: - description: Minimum air temperature during the hour - dtype: float32 - long_name: minimum_air_temperature - units: K - downward_shortwave_irradiance_at_earth_surface: - description: Average global solar radiation - dtype: float32 - flag: solarad_flag - long_name: downward_shortwave_irradiance_at_earth_surface - units: W m-2 - hourly_maximum_downward_shortwave_irradiance_at_earth_surface: - description: Maximum global solar radiation - dtype: float32 - flag: solarad_max_flag - long_name: maximum_downward_shortwave_irradiance_at_earth_surface - units: W m-2 - hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for maximum global solar radiation. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag - hourly_minimum_downward_shortwave_irradiance_at_earth_surface: - description: Minimum global solar radiation - dtype: float32 - flag: solarad_min_flag - long_name: minimum_downward_shortwave_irradiance_at_earth_surface - units: W m-2 - hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for minimum global solar radiation. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag - downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for average global solar radiation. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: downward_shortwave_irradiance_at_earth_surface - location_latitude: - description: Latitude of the station (deg. North) - dtype: float32 - long_name: latitude - units: degree_north - location_longitude: - description: Longitude of the station (deg. East) - dtype: float32 - long_name: longitude - units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn - relative_humidity: - description: Hourly relative humidity is computed from 5-minute averages in almost all cases, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. - dtype: float32 - flag: rh_hr_avg_flag - long_name: average_relative_humidity - units: '%' - relative_humidity_quality_flag: - description: QC flag for RH average. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: average_relative_humidity_quality_flag - report_id: - description: Identifier in the USCRN meta-database - long_name: id - report_timestamp: - description: observation date time UTC - dtype: datetime64[ns] - long_name: date_of_observation - soil_moisture_100cm_from_earth_surface: - description: Average soil moisture at 100 cm below the surface - dtype: float32 - long_name: soil_moisture_100cm_from_earth_surface - units: m3 m-3 - soil_moisture_10cm_from_earth_surface: - description: Average soil moisture at 10 cm below the surface - dtype: float32 - long_name: soil_moisture_10cm_from_earth_surface - units: m3 m-3 - soil_moisture_20cm_from_earth_surface: - description: Average soil moisture at 20 cm below the surface - dtype: float32 - long_name: soil_moisture_20cm_from_earth_surface - units: m3 m-3 - soil_moisture_50cm_from_earth_surface: - description: Average soil moisture at 50 cm below the surface - dtype: float32 - long_name: soil_moisture_50cm_from_earth_surface - units: m3 m-3 - soil_moisture_5cm_from_earth_surface: - description: Average soil moisture at 5 cm below the surface - dtype: float32 - long_name: soil_moisture_5cm_from_earth_surface - units: m3 m-3 - soil_temperature: - description: On 2013-01-07 at 1500 UTC, USCRN began reporting corrected surface temperature measurements for some stations. These changes impact previous users of the data because the corrected values differ from uncorrected values. To distinguish between uncorrected (raw) and corrected surface temperature measurements, a surface temperature type field was added to the data product. The possible values of the this field are "R" to denote raw surface temperature measurements, "C" to denote corrected surface temperature measurements, and "U" for unknown/missing. - dtype: float32 - flag: sur_temp_flag - long_name: average_soil_temperature - processing_level: sur_temp_type - units: K - soil_temperature_100cm_from_earth_surface: - description: Average soil temperature at 100 cm below the surface - dtype: float32 - long_name: soil_temperature_100cm_from_earth_surface - units: K - soil_temperature_10cm_from_earth_surface: - description: Average soil temperature at 10 cm below the surface - dtype: float32 - long_name: soil_temperature_10cm_from_earth_surface - units: K - soil_temperature_20cm_from_earth_surface: - description: Average soil temperature at 20 cm below the surface - dtype: float32 - long_name: soil_temperature_20cm_from_earth_surface - units: K - soil_temperature_50cm_from_earth_surface: - description: Average soil temperature at 50 cm below the surface - dtype: float32 - long_name: soil_temperature_50cm_from_earth_surface - units: K - soil_temperature_5cm_from_earth_surface: - description: Average soil temperature at 5 cm below the surface - dtype: float32 - long_name: soil_temperature_5cm_from_earth_surface - units: K - hourly_maximum_soil_temperature: - description: Maximum infrared surface temperature - dtype: float32 - flag: sur_temp_max_flag - long_name: maximum_soil_temperature - units: K - hourly_maximum_soil_temperature_flag: - description: QC flag for infrared surface temperature maximum. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: maximum_soil_temperature_quality_flag - hourly_minimum_soil_temperature: - description: Minimum infrared surface temperature - dtype: float32 - flag: sur_temp_min_flag - long_name: minimum_soil_temperature - units: K - hourly_minimum_soil_temperature_quality_flag: - description: QC flag for infrared surface temperature minimum. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: minimum_soil_temperature_quality_flag - soil_temperature_processing_level: - description: Type of infrared surface temperature measurement - dtype: float32 - long_name: soil_temperature_processing_level - soil_temperature_quality_flag: - description: QC flag for infrared surface temperature. 0 denotes good data and 3 denotes erroneous data. - dtype: float32 - long_name: average_soil_temperature_quality_flag - primary_station_id: - description: Station identification code - dtype: object - long_name: wbanno - mandatory_columns: - - alternative_name - - primary_station_id - - longitude|station_configuration - - latitude|station_configuration - - report_timestamp - order_by: - - primary_station_id - - report_timestamp - products: - - columns: - - t - - t_max - - t_min - - p_calc - - solarad - - solarad_max - - solarad_min - - sur_temp - - sur_temp_max - - sur_temp_min - - rh_hr_avg - - soil_moisture_5 - - soil_moisture_10 - - soil_moisture_20 - - soil_moisture_50 - - soil_moisture_100 - - soil_temp_5 - - soil_temp_10 - - soil_temp_20 - - soil_temp_50 - - soil_temp_100 - group_name: variables - - columns: - - random_pm - group_name: random_uncertainty - - columns: - - terr_p - group_name: positive_total_uncertainty - - columns: - - terr_m - group_name: negative_total_uncertainty - - columns: - - tmaxerr_p - group_name: max_positive_total_uncertainty - - columns: - - tmaxerr_m - group_name: max_negative_total_uncertainty - - columns: - - tminerr_p - group_name: min_positive_total_uncertainty - - columns: - - tminerr_m - group_name: min_negative_total_uncertainty - - columns: - - sys_p - group_name: positive_systematic_uncertainty - - columns: - - sys_m - group_name: negative_systematic_uncertainty - - columns: - - quasisys_p - group_name: positive_quasisystematic_uncertainty - - columns: - - quasisys_m - group_name: negative_quasisystematic_uncertainty - - columns: - - solarad_flag - - solarad_max_flag - - solarad_min_flag - - sur_temp_flag - - sur_temp_max_flag - - sur_temp_min_flag - - rh_hr_avg_flag - group_name: flag - - columns: - - sur_temp_type - group_name: processing_level - USCRN_MONTHLY: - cdm_mapping: - melt_columns: true - rename: - crx_vn_monthly: logbook_version - date_of_observation: report_timestamp - id: report_id - latitude: latitude|station_configuration - longitude: longitude|station_configuration - p_monthly_calc: accumulated_precipitation - quasisys_m: air_temperature_negative_quasisystematic_uncertainty - quasisys_p: air_temperature_positive_quasisystematic_uncertainty - random_m: air_temperature_negative_random_uncertainty - random_p: air_temperature_positive_random_uncertainty - sitename: alternative_name - sur_temp_monthly_avg: soil_temperature - sur_temp_monthly_max: monthly_maximum_soil_temperature - sur_temp_monthly_min: monthly_minimum_soil_temperature - sur_temp_monthly_type: soil_temperature_processing_level - sys_m: air_temperature_negative_systematic_uncertainty - sys_p: air_temperature_positive_systematic_uncertainty - t_monthly_max: daily_maximum_air_temperature - t_monthly_mean: air_temperature - t_monthly_min: daily_minimum_air_temperature - terr_m: air_temperature_negative_total_uncertainty - terr_p: air_temperature_positive_total_uncertainty - tmaxerr_m: air_temperature_max_negative_total_uncertainty - tmaxerr_p: air_temperature_max_positive_total_uncertainty - tminerr_m: air_temperature_min_negative_total_uncertainty - tminerr_p: air_temperature_min_positive_total_uncertainty - wbanno: primary_station_id - unit_changes: - accumulated_precipitation: - names: - mm: mm - offset: 0 - scale: 1 - data_table: unc_monthly - descriptions: - accumulated_precipitation: - description: The total amount of precipitation - dtype: float32 - long_name: accumulated_precipitation - units: mm - air_temperature: - description: The mean air temperature calculated using the typical historical approach of (T_MONTHLY_MAX + T_MONTHLY_MIN) / 2 - dtype: float32 - long_name: mean_air_temperature - units: K - air_temperature_max_negative_total_uncertainty: - description: negative uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_negative_total_uncertainty - units: K - air_temperature_max_positive_total_uncertainty: - description: positive uncertainties in tmax which are related to temperature - dtype: float32 - long_name: air_temperature_max_positive_total_uncertainty - units: K - air_temperature_min_negative_total_uncertainty: - description: negative uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_negative_total_uncertainty - units: K - air_temperature_min_positive_total_uncertainty: - description: positive uncertainties in tmin which are related to temperature - dtype: float32 - long_name: minimum_air_temperature_positive_total_uncertainty - units: K - air_temperature_negative_quasisystematic_uncertainty: - description: -ve/-ve Quasi-systematic - dtype: float32 - long_name: air_temperature_negative_quasisystematic_uncertainty - units: K - air_temperature_negative_random_uncertainty: - description: -ve Random uncertainty - dtype: float32 - long_name: air_temperature_negative_random_uncertainty - units: K - air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty - units: K - air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature - dtype: float32 - long_name: air_temperature_negative_total_uncertainty - units: K - air_temperature_positive_quasisystematic_uncertainty: - description: +ve Quasi-systematic uncertainty - dtype: float32 - long_name: air_temperature_positive_quasisystematic_uncertainty - units: K - air_temperature_positive_random_uncertainty: - description: +ve Random uncertainty - dtype: float32 - long_name: air_temperature_positive_random_uncertainty - units: K - air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty - units: K - air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature - dtype: float32 - long_name: air_temperature_positive_total_uncertainty - units: K - alternative_name: - description: Alternative name for station - dtype: object - long_name: sitename - daily_maximum_air_temperature: - description: The maximum air temperature - dtype: float32 - long_name: air_temperature_max - negative_total_uncertainty: tmaxerr_m - positive_total_uncertainty: tmaxerr_p - units: K - daily_minimum_air_temperature: - description: The minimum air temperature - dtype: float32 - long_name: minimum_air_temperature - negative_total_uncertainty: tminerr_m - positive_total_uncertainty: tminerr_p - units: K - location_latitude: - description: Station latitude, using WGS-84, with a precision of 4 decimal places - dtype: float32 - long_name: latitude - units: degree_north - location_longitude: - description: Station longitude, using WGS-84, with a precision of 4 decimal places - dtype: float32 - long_name: longitude - units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn_monthly - report_id: - description: Identifier in the USCRN meta-database - long_name: id - report_timestamp: - description: observation date time UTC - dtype: datetime64[ns] - long_name: date_of_observation - soil_temperature: - description: Monthly maximum/minimum/average surface temperatures are the average of all available daily max/min/avg values. To be considered valid, there must be fewer than 4 consecutive values missing, and no more than 5 total values missing. - dtype: float32 - long_name: average_soil_temperature - processing_level: sur_temp_monthly_type - units: K - monthly_maximum_soil_temperature: - description: The maximum infrared surface temperature - dtype: float32 - long_name: maximum_soil_temperature - units: K - monthly_minimum_soil_temperature: - description: The minimum infrared surface temperature - dtype: float32 - long_name: minimum_soil_temperature - units: K - soil_temperature_processing_level: - description: Type of infrared surface temperature measurement - dtype: float32 - long_name: soil_temperature_processing_level - primary_station_id: - description: Station identification code - dtype: object - long_name: wbanno - mandatory_columns: - - alternative_name - - primary_station_id - - longitude|station_configuration - - latitude|station_configuration - - report_timestamp - order_by: - - primary_station_id - - report_timestamp - products: - - columns: - - t_monthly_max - - t_monthly_min - - t_monthly_mean - - p_monthly_calc - - sur_temp_monthly_max - - sur_temp_monthly_min - - sur_temp_monthly_avg - group_name: variables - - columns: - - random_p - group_name: positive_random_uncertainty - - columns: - - random_m - group_name: negative_random_uncertainty - - columns: - - terr_p - - tmaxerr_p - - tminerr_p - group_name: positive_total_uncertainty - - columns: - - terr_m - - tmaxerr_m - - tminerr_m - group_name: negative_total_uncertainty - - columns: - - quasisys_p - group_name: positive_quasisystematic_uncertainty - - columns: - - quasisys_m - group_name: negative_quasisystematic_uncertainty - - columns: - - sys_p - group_name: positive_systematic_uncertainty - - columns: - - sys_m - group_name: negative_systematic_uncertainty - - columns: - - sur_temp_monthly_type - group_name: processing_level - USCRN_SUBHOURLY: - cdm_mapping: - melt_columns: true - rename: - crx_vn: logbook_version - date_of_observation: report_timestamp - id: report_id - latitude: latitude|station_configuration - longitude: longitude|station_configuration - precipitation: accumulated_precipitation - quasisys_pm: air_temperature_quasisystematic_uncertainty - random_pm: air_temperature_random_uncertainty - relative_humidity: relative_humidity - rh_flag: relative_humidity_quality_flag - sitename: alternative_name - soil_moisture_5: soil_moisture_5cm_from_earth_surface - soil_temperature_5: soil_temperature_5cm_from_earth_surface - solar_radiation: downward_shortwave_irradiance_at_earth_surface - sr_flag: downward_shortwave_irradiance_at_earth_surface_quality_flag - st_flag: soil_temperature_processing_level_quality_flag - st_type: soil_temperature_processing_level - surface_temperature: soil_temperature - sys_m: air_temperature_negative_systematic_uncertainty - sys_p: air_temperature_positive_systematic_uncertainty - t: air_temperature - terr_m: air_temperature_negative_total_uncertainty - terr_p: air_temperature_positive_total_uncertainty - wbanno: primary_station_id - wetness: wetness - wet_flag: wetness_quality_flag - wind_1_5: wind_speed_2_meters_from_earth_surface - wind_flag: wind_speed_2_meters_from_earth_surface_quality_flag - unit_changes: - accumulated_precipitation: - names: - mm: mm - offset: 0 - scale: 1 - data_table: unc_subhourly - descriptions: - accumulated_precipitation: - description: Total amount of precipitation - dtype: float32 - long_name: accumulated_precipitation - units: mm - air_temperature: - description: air temperature - dtype: float32 - long_name: average_air_temperature - negative_systematic_uncertainty: sys_m - negative_total_uncertainty: terr_m - positive_systematic_uncertainty: sys_p - positive_total_uncertainty: terr_p - quasisystematic_uncertainty: quasisys_pm - random_uncertainty: random_pm - units: K - air_temperature_negative_systematic_uncertainty: - description: -ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_negative_systematic_uncertainty - units: K - air_temperature_negative_total_uncertainty: - description: negative uncertainty of temperature - dtype: float32 - long_name: air_temperature_negative_total_uncertainty - units: K - air_temperature_positive_systematic_uncertainty: - description: +ve Systematic uncertainty - dtype: float32 - long_name: air_temperature_positive_systematic_uncertainty - units: K - air_temperature_positive_total_uncertainty: - description: positive uncertainty of temperature - dtype: float32 - long_name: air_temperature_positive_total_uncertainty - units: K - air_temperature_quasisystematic_uncertainty: - description: +ve/-ve Quasi-systematic - dtype: float32 - long_name: air_temperature_quasisystematic_uncertainty - units: K - air_temperature_random_uncertainty: - description: +ve/-ve Random - dtype: float32 - long_name: air_temperature_random_uncertainty - units: K - alternative_name: - description: Alternative name for station - dtype: object - long_name: sitename - downward_shortwave_irradiance_at_earth_surface: - description: Average global solar radiation received - dtype: float32 - flag: sr_flag - long_name: downward_shortwave_irradiance_at_earth_surface - units: W m-2 - downward_shortwave_irradiance_at_earth_surface_quality_flag: - description: QC flag for the average global solar radiation measurement - dtype: float32 - long_name: downward_shortwave_irradiance_at_earth_surface_quality_flag - location_latitude: - description: Latitude of the station (deg. North) - dtype: float32 - long_name: latitude - units: degree_north - location_longitude: - description: Longitude of the station (deg. East) - dtype: float32 - long_name: longitude - units: degree_east - logbook_version: - description: The version number of the station datalogger program that was in effect at the time of the observation - dtype: object - long_name: crx_vn - relative_humidity: - description: All USCRN stations now report 5-minute relative humidity averages, however the two Asheville, NC stations reported only hourly RH values until 2007-02-22. - dtype: float32 - flag: rh_flag - long_name: average_relative_humidity - units: '%' - relative_humidity_quality_flag: - description: QC flag for the relative humidity measurement - dtype: float32 - long_name: average_relative_humidity_quality_flag - report_id: - description: Identifier in the USCRN meta-database - long_name: id - report_timestamp: - description: observation date time UTC - dtype: datetime64[ns] - long_name: date_of_observation - soil_moisture_5cm_from_earth_surface: - description: Average soil moisture at 5 cm below the surface - dtype: float32 - long_name: soil_moisture_5cm_from_earth_surface - units: m3 m-3 - soil_temperature: - description: USCRN stations have multiple co-located soil sensors that record independent measurements. The soil values reported in this dataset are calculated from these multiple independent measurements. Soil moisture is the ratio of water volume over sample volume (m^3 water m^-3 soil). - dtype: float32 - flag: st_flag - long_name: average_soil_temperature - processing_level: st_type - units: K - soil_temperature_5cm_from_earth_surface: - description: Average soil temperature at 5 cm below the surface - dtype: float32 - long_name: soil_temperature_5cm_from_earth_surface - units: K - soil_temperature_processing_level: - description: The type of infrared surface temperature measurement - dtype: float32 - long_name: soil_temperature_processing_level - soil_temperature_processing_level_quality_flag: - description: QC flag for the surface temperature measurement - dtype: float32 - long_name: soil_temperature_processing_level_quality_flag - primary_station_id: - description: Station identification code - dtype: object - long_name: wbanno - wetness: - description: This parameter indicates the presence or absence of moisture due to precipitation, in Ohms. High values (>= 1000) indicate an absence of moisture. Low values (< 1000) indicate the presence of moisture. - dtype: float32 - long_name: wetness - quality_flag: wet_flag - units: Ohms - wetness_quality_flag: - description: QC flag for the wetness measurement - dtype: float32 - long_name: wetness_quality_flag - wind_speed_2_meters_from_earth_surface: - description: Average wind speed, in meters per second, at a height of 1.5 meters - dtype: float32 - flag: wind_flag - long_name: wind_speed_2_meters_from_earth_surface - units: m s-1 - wind_speed_2_meters_from_earth_surface_quality_flag: - description: QC flag for the wind speed measurement - dtype: float32 - long_name: wind_speed_2_meters_from_earth_surface_quality_flag - mandatory_columns: - - alternative_name - - primary_station_id - - longitude|station_configuration - - latitude|station_configuration - - report_timestamp - order_by: - - primary_station_id - - report_timestamp - products: - - columns: - - t - - precipitation - - solar_radiation - - surface_temperature - - relative_humidity - - soil_moisture_5 - - soil_temperature_5 - - wetness - - wind_1_5 - group_name: variables - - columns: - - terr_m - group_name: negative_total_uncertainty - - columns: - - random_pm - group_name: random_uncertainty - - columns: - - terr_p - group_name: positive_total_uncertainty - - columns: - - quasisys_pm - group_name: quasisystematic_uncertainty - - columns: - - sys_p - group_name: positive_systematic_uncertainty - - columns: - - sys_m - group_name: negative_systematic_uncertainty - - columns: - - sr_flag - - st_flag - - rh_flag - - wet_flag - - wind_flag - group_name: flag - - columns: - - st_type - group_name: processing_level -space_columns: - y: latitude|station_configuration - x: longitude|station_configuration diff --git a/tests/upgrade_service_definition.py b/tests/upgrade_service_definition.py index bcbf163..56bfb93 100644 --- a/tests/upgrade_service_definition.py +++ b/tests/upgrade_service_definition.py @@ -85,7 +85,7 @@ def main(old_path): new_header_columns = [list(hc.values())[0] for hc in header_columns] new_data["sources"][source]["header_columns"] = new_header_columns # Dump to YAML - output_path = Path(str(old_path).replace(".json", ".yml")) + output_path = Path((str(old_path) + "new").replace(".json", ".yml")) with output_path.open("w") as op: op.write(yaml.dump(new_data)) From a5764a3c478868da4bff74f39f07745f003a5769 Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 10:55:47 +0200 Subject: [PATCH 04/20] fixes for new USCRN service definition --- .../service_definition.yml | 61 ++++++++++--------- tests/conftest.py | 7 ++- tests/test_api.py | 58 +++++++++--------- 3 files changed, 68 insertions(+), 58 deletions(-) diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml index eafd83e..8e2e2ff 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml @@ -3,7 +3,7 @@ global_attributes: licence_list: 20180314_Copernicus_License_V1.1 responsible_organisation: ECMWF out_columns_order: -- station_name +- primary_station_id - alternative_name - report_timestamp - report_id @@ -121,12 +121,12 @@ sources: t_daily_min: daily_minimum_air_temperature terr_m: air_temperature_negative_total_uncertainty terr_p: air_temperature_positive_total_uncertainty - tmaxerr_m: air_temperature_max_negative_total_uncertainty - tmaxerr_p: air_temperature_max_positive_total_uncertainty - tmeanerr_m: air_temperature_mean_negative_total_uncertainty - tmeanerr_p: air_temperature_mean_positive_total_uncertainty - tminerr_m: air_temperature_min_negative_total_uncertainty - tminerr_p: air_temperature_min_positive_total_uncertainty + tmaxerr_m: daily_maximum_air_temperature_negative_total_uncertainty + tmaxerr_p: daily_maximum_air_temperature_positive_total_uncertainty + tmeanerr_m: mean_air_temperature_negative_total_uncertainty + tmeanerr_p: mean_air_temperature_positive_total_uncertainty + tminerr_m: daily_minimum_air_temperature_negative_total_uncertainty + tminerr_p: daily_minimum_air_temperature_positive_total_uncertainty wbanno: primary_station_id wind_1_5: wind_speed_2_meters_from_earth_surface wind_flag: wind_speed_2_meters_from_earth_surface_quality_flag @@ -203,7 +203,7 @@ sources: dtype: float32 long_name: solarad_daily units: MJ m-2 - latitude: + latitude|station_configuration: description: Latitude of the measurement station, -90 to 90. dtype: float32 long_name: latitude @@ -212,7 +212,7 @@ sources: description: LogBook software and version. dtype: float32 long_name: crx_vn - longitude: + longitude|station_configuration: description: Longitude of the measurement station, -180.0 to 180.0. dtype: float32 long_name: longitude @@ -361,7 +361,7 @@ sources: description: 'This parameter indicates the level of processing applied to the soil temperature measurement: raw data (''R''), corrected data (''C''). A letter ''U'' indicates that this information is unknown.' dtype: float32 long_name: sur_temp_daily_type - station_name: + primary_station_id: description: This is the station identification code. dtype: object long_name: wbanno @@ -478,10 +478,10 @@ sources: t_min: daily_minimum_air_temperature terr_m: air_temperature_negative_total_uncertainty terr_p: air_temperature_positive_total_uncertainty - tmaxerr_m: air_temperature_max_negative_total_uncertainty - tmaxerr_p: air_temperature_max_positive_total_uncertainty - tminerr_m: air_temperature_min_negative_total_uncertainty - tminerr_p: air_temperature_min_positive_total_uncertainty + tmaxerr_m: daily_maximum_air_temperature_negative_total_uncertainty + tmaxerr_p: daily_maximum_air_temperature_positive_total_uncertainty + tminerr_m: daily_minimum_air_temperature_negative_total_uncertainty + tminerr_p: daily_minimum_air_temperature_positive_total_uncertainty wbanno: primary_station_id unit_changes: accumulated_precipitation: @@ -577,7 +577,7 @@ sources: description: Quality indicator variable 'downward_shortwave_irradiance_at_earth_surface'. 0 denotes good data and 3 denotes erroneous data. dtype: float32 long_name: solarad_flag - latitude: + latitude|station_configuration: description: Latitude of the measurement station, -90 to 90. dtype: float32 long_name: latitude @@ -586,7 +586,7 @@ sources: description: The version number of the station datalogger program that was in effect at the time of the observation. dtype: float32 long_name: crx_vn - longitude: + longitude|station_configuration: description: Longitude of the measurement station, -180.0 to 180.0. dtype: float32 long_name: longitude @@ -728,18 +728,18 @@ sources: description: This parameter indicates if the "Soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). dtype: float32 long_name: sur_temp_flag - station_name: + primary_station_id: description: This is the station identification code. - dtype: float32 + dtype: object long_name: wbanno mandatory_columns: - alternative_name - - station_name + - primary_station_id - longitude - latitude - report_timestamp order_by: - - station_name + - primary_station_id - report_timestamp products: - columns: @@ -838,10 +838,10 @@ sources: t_monthly_min: daily_minimum_air_temperature terr_m: air_temperature_negative_total_uncertainty terr_p: air_temperature_positive_total_uncertainty - tmaxerr_m: air_temperature_max_negative_total_uncertainty - tmaxerr_p: air_temperature_max_positive_total_uncertainty - tminerr_m: air_temperature_min_negative_total_uncertainty - tminerr_p: air_temperature_min_positive_total_uncertainty + tmaxerr_m: daily_maximum_air_temperature_total_uncertainty + tmaxerr_p: daily_maximum_air_temperature_positive_total_uncertainty + tminerr_m: daily_minimum_air_temperature_negative_total_uncertainty + tminerr_p: daily_minimum_air_temperature_positive_total_uncertainty wbanno: primary_station_id unit_changes: accumulated_precipitation: @@ -913,7 +913,7 @@ sources: description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object long_name: sitename - latitude: + latitude|station_configuration: description: Latitude of the measurement station, -90 to 90. dtype: float32 long_name: latitude @@ -922,7 +922,7 @@ sources: description: LogBook software and version. dtype: float32 long_name: crx_vn_monthly - longitude: + longitude|station_configuration: description: Longitude of the measurement station, -180.0 to 180.0. dtype: float32 long_name: longitude @@ -1145,6 +1145,11 @@ sources: dtype: float32 long_name: random_pm units: K + air_temperature_quasisystematic_uncertainty: + description: "For \"Air temperature\" (described in this list), this parameter is one of the uncertainty sources contributing to estimating the measurement error (see detailed description of \"Positive quasisystematic uncertainty\" in the table \"Related variables\"). This parameter has the same unit as the measurand it refers to, i.e. Kelvin (K). However, because this parameter represents a difference in temperature space, it can also be interpreted directly as being in degrees Celsius (°C)." + dtype: float32 + long_name: quasisys_pm + units: K alternative_name: description: This describes the location of the station where measurements were taken. There are four elements separated by an underscore. The first element (two letters) indicates the State, the second element indicates the nearest city, and the last two elements describe a vector from the city to the station. The distance is given in statute miles (1 statute mile = 1609.344 metres) and the bearing is given in the 16-element compass rose. For example, station "VA_Charlottesville_2_SSE" refers to a station located 2 miles South-South-East of Charlottesville, Virginia. dtype: object @@ -1159,7 +1164,7 @@ sources: description: Quality indicator variable 'downward_shortwave_irradiance_at_earth_surface'. 0 denotes good data and 3 denotes erroneous data. dtype: float32 long_name: sr_flag - latitude: + latitude|station_configuration: description: Latitude of the measurement station, -90 to 90. dtype: float32 long_name: latitude @@ -1168,7 +1173,7 @@ sources: description: LogBook software and version. dtype: float32 long_name: crx_vn - longitude: + longitude|station_configuration: description: Longitude of the measurement station, -180.0 to 180.0. dtype: float32 long_name: longitude diff --git a/tests/conftest.py b/tests/conftest.py index 6353d0e..a177ff2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,7 +127,12 @@ def test_repository(test_session, test_s3_client, test_config): "insitu-observations-near-surface-temperature-us-climate-reference-network" ) service_definition = get_service_definition(dataset_name) - for dataset_source in ["uscrn_daily", "uscrn_subhourly"]: + for dataset_source in [ + "uscrn_daily", + "uscrn_subhourly", + "uscrn_hourly", + "uscrn_monthly", + ]: start_year, end_year = get_test_years(dataset_source) run_ingestion_pipeline( dataset_name, diff --git a/tests/test_api.py b/tests/test_api.py index 821e7d4..6f70535 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -13,20 +13,20 @@ @pytest.mark.parametrize( "dataset_name,source,test_update", [ - # ( - # "insitu-observations-woudc-ozone-total-column-and-profiles", - # "OzoneSonde", - # False, - # ), - # ( - # "insitu-observations-woudc-ozone-total-column-and-profiles", - # "TotalOzone", - # False, - # ), - # ("insitu-observations-igra-baseline-network", "IGRA", False), - # ("insitu-observations-igra-baseline-network", "IGRA_H", False), - # ("insitu-comprehensive-upper-air-observation-network", "CUON", True), - # ("insitu-observations-gruan-reference-network", "GRUAN", False), + ( + "insitu-observations-woudc-ozone-total-column-and-profiles", + "OzoneSonde", + False, + ), + ( + "insitu-observations-woudc-ozone-total-column-and-profiles", + "TotalOzone", + False, + ), + ("insitu-observations-igra-baseline-network", "IGRA", False), + ("insitu-observations-igra-baseline-network", "IGRA_H", False), + ("insitu-comprehensive-upper-air-observation-network", "CUON", True), + ("insitu-observations-gruan-reference-network", "GRUAN", False), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", "uscrn_subhourly", @@ -47,21 +47,21 @@ "uscrn_monthly", False, ), - # ( - # "insitu-observations-gnss", - # "IGS", - # False, - # ), - # ( - # "insitu-observations-gnss", - # "EPN", - # False, - # ), - # ( - # "insitu-observations-gnss", - # "IGS_R3", - # False, - # ), + ( + "insitu-observations-gnss", + "IGS", + False, + ), + ( + "insitu-observations-gnss", + "EPN", + False, + ), + ( + "insitu-observations-gnss", + "IGS_R3", + False, + ), ], ) def test_run_ingestion_pipeline( From 8927e29a41e374262d65c11273163eb1399dbb83 Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 11:08:43 +0200 Subject: [PATCH 05/20] fix CI --- .github/workflows/on-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 5a7f247..f4b2fce 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -36,7 +36,7 @@ jobs: uses: actions/checkout@v3 with: repository: ecmwf-projects/cdm-obs.git - ref: 'cadsobs-changes' + ref: 'new-variables' path: common_data_model - name: Deploy test ingestion database env: From 278e56804d77318856d27c507484326bfbe87a7d Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 12:49:10 +0200 Subject: [PATCH 06/20] tests fixed --- cdsobs/cdm/lite.py | 2 +- cdsobs/constants.py | 16 ---- .../service_definition.yml | 24 +++--- tests/test_cdm_api.py | 79 +++++++++---------- tests/test_http_api.py | 41 ++++++++-- 5 files changed, 85 insertions(+), 77 deletions(-) diff --git a/cdsobs/cdm/lite.py b/cdsobs/cdm/lite.py index 340a470..dbb6628 100644 --- a/cdsobs/cdm/lite.py +++ b/cdsobs/cdm/lite.py @@ -73,7 +73,7 @@ "quasisystematic_uncertainty", "positive_quasisystematic_uncertainty", "negative_quasisystematic_uncertainty", - "flag", + "quality_flag", ] cdm_lite_variables = dict( mandatory=variable_names, diff --git a/cdsobs/constants.py b/cdsobs/constants.py index d3d8865..a0d7bba 100644 --- a/cdsobs/constants.py +++ b/cdsobs/constants.py @@ -106,19 +106,3 @@ "EPN": ["precipitable_water_column"], }, } -AUX_FIELDS = [ - "total_uncertainty", - "positive_total_uncertainty", - "negative_total_uncertainty", - "max_positive_total_uncertainty", - "max_negative_total_uncertainty", - "min_positive_total_uncertainty", - "min_negative_total_uncertainty", - "random_uncertainty", - "positive_systematic_uncertainty", - "negative_systematic_uncertainty", - "quasisystematic_uncertainty", - "positive_quasisystematic_uncertainty", - "negative_quasisystematic_uncertainty", - "flag", -] diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml index 8e2e2ff..1584ab4 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml @@ -466,10 +466,10 @@ sources: solarad_min_flag: minimum_solar_irradiance_quality_flag sur_temp: soil_temperature sur_temp_flag: soil_temperature_quality_flag - sur_temp_max: hourly_maximum_soil_temperature - sur_temp_max_flag: hourly_maximum_soil_temperature_flag - sur_temp_min: hourly_minimum_soil_temperature - sur_temp_min_flag: hourly_minimum_soil_temperature_quality_flag + sur_temp_max: maximum_soil_temperature + sur_temp_max_flag: maximum_soil_temperature_flag + sur_temp_min: minimum_soil_temperature + sur_temp_min_flag: minimum_soil_temperature_quality_flag sur_temp_type: soil_temperature_processing_level sys_m: air_temperature_negative_systematic_uncertainty sys_p: air_temperature_positive_systematic_uncertainty @@ -608,13 +608,13 @@ sources: dtype: float32 long_name: tmaxerr_p units: K - hourly_maximum_soil_temperature: + maximum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the highest value measured during the aggregation period. dtype: float32 long_name: sur_temp_max quality_flag: sur_temp_max_flag units: K - hourly_maximum_soil_temperature_quality_flag: + maximum_soil_temperature_quality_flag: description: This parameter indicates if the "Maximum soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). denotes good data and 3 denotes erroneous data. dtype: float32 long_name: sur_temp_max_flag @@ -635,13 +635,13 @@ sources: dtype: float32 long_name: tminerr_p units: K - hourly_minimum_soil_temperature: + minimum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the lowest value measured during the aggregation period. dtype: float32 long_name: sur_temp_min quality_flag: sur_temp_min_flag units: K - hourly_minimum_soil_temperature_quality_flag: + minimum_soil_temperature_quality_flag: description: This parameter indicates if the "Minimum soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). dtype: float32 long_name: sur_temp_min_flag @@ -827,8 +827,8 @@ sources: sitename: alternative_name solrad_monthly_avg: monthly_global_solar_radiation sur_temp_monthly_avg: soil_temperature - sur_temp_monthly_max: monthly_maximum_soil_temperature - sur_temp_monthly_min: monthly_minimum_soil_temperature + sur_temp_monthly_max: maximum_soil_temperature + sur_temp_monthly_min: minimum_soil_temperature sur_temp_monthly_type: soil_temperature_processing_level sys_m: air_temperature_negative_systematic_uncertainty sys_p: air_temperature_positive_systematic_uncertainty @@ -944,7 +944,7 @@ sources: dtype: float32 long_name: tmaxerr_p units: K - monthly_maximum_soil_temperature: + maximum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the highest value measured during the aggregation period. dtype: float32 long_name: sur_temp_monthly_max @@ -971,7 +971,7 @@ sources: dtype: float32 long_name: tminerr_p units: K - monthly_minimum_soil_temperature: + minimum_soil_temperature: description: From measurements of "soil temperature" (described in this list), this parameter provides the lowest value measured during the aggregation period. dtype: float32 long_name: sur_temp_monthly_min diff --git a/tests/test_cdm_api.py b/tests/test_cdm_api.py index 674a655..c2b0181 100644 --- a/tests/test_cdm_api.py +++ b/tests/test_cdm_api.py @@ -82,35 +82,32 @@ def test_get_aux_fields_mapping_from_service_definition(): ], "daily_maximum_air_temperature": [], "daily_minimum_air_temperature": [], - "downward_shortwave_irradiance_at_earth_surface": [ - { - "auxvar": "downward_shortwave_irradiance_at_earth_surface_quality_flag", - "metadata_name": "flag", - } + "maximum_soil_temperature": [ + {"auxvar": "maximum_soil_temperature_flag", "metadata_name": "quality_flag"} ], - "hourly_maximum_downward_shortwave_irradiance_at_earth_surface": [ + "maximum_solar_irradiance": [ { - "auxvar": "hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "metadata_name": "flag", + "auxvar": "maximum_solar_irradiance_quality_flag", + "metadata_name": "quality_flag", } ], - "hourly_maximum_soil_temperature": [ - {"auxvar": "hourly_maximum_soil_temperature_flag", "metadata_name": "flag"} - ], - "hourly_minimum_downward_shortwave_irradiance_at_earth_surface": [ + "minimum_soil_temperature": [ { - "auxvar": "hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "metadata_name": "flag", + "auxvar": "minimum_soil_temperature_quality_flag", + "metadata_name": "quality_flag", } ], - "hourly_minimum_soil_temperature": [ + "minimum_solar_irradiance": [ { - "auxvar": "hourly_minimum_soil_temperature_quality_flag", - "metadata_name": "flag", + "auxvar": "minimum_solar_irradiance_quality_flag", + "metadata_name": "quality_flag", } ], "relative_humidity": [ - {"auxvar": "relative_humidity_quality_flag", "metadata_name": "flag"} + { + "auxvar": "relative_humidity_quality_flag", + "metadata_name": "quality_flag", + } ], "soil_moisture_100cm_from_earth_surface": [], "soil_moisture_10cm_from_earth_surface": [], @@ -118,15 +115,17 @@ def test_get_aux_fields_mapping_from_service_definition(): "soil_moisture_50cm_from_earth_surface": [], "soil_moisture_5cm_from_earth_surface": [], "soil_temperature": [ - {"auxvar": "soil_temperature_quality_flag", "metadata_name": "flag"} + {"auxvar": "soil_temperature_quality_flag", "metadata_name": "quality_flag"} ], "soil_temperature_100cm_from_earth_surface": [], "soil_temperature_10cm_from_earth_surface": [], "soil_temperature_20cm_from_earth_surface": [], "soil_temperature_50cm_from_earth_surface": [], "soil_temperature_5cm_from_earth_surface": [], + "solar_irradiance": [ + {"auxvar": "solar_irradiance_quality_flag", "metadata_name": "quality_flag"} + ], } - service_definition = get_service_definition(dataset_name) source_definition = service_definition.sources[source] variables = get_variables_from_service_definition(service_definition, source) @@ -142,44 +141,40 @@ def test_get_aux_fields_mapping_from_service_definition(): "air_temperature_negative_systematic_uncertainty", "air_temperature_positive_quasisystematic_uncertainty", "air_temperature_negative_quasisystematic_uncertainty", - "downward_shortwave_irradiance_at_earth_surface_quality_flag", - "hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "hourly_maximum_soil_temperature_flag", - "hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "hourly_minimum_soil_temperature_quality_flag", + "maximum_soil_temperature_flag", + "maximum_solar_irradiance_quality_flag", + "minimum_soil_temperature_quality_flag", + "minimum_solar_irradiance_quality_flag", "relative_humidity_quality_flag", "soil_temperature_quality_flag", + "solar_irradiance_quality_flag", ] assert not actual.var_has_uncertainty_field("accumulated_precipitation") assert actual.var_has_uncertainty_field("air_temperature") assert actual.vars_with_uncertainty_field == ["air_temperature"] assert actual.quality_flag_fields == [ - "downward_shortwave_irradiance_at_earth_surface_quality_flag", - "hourly_maximum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "hourly_maximum_soil_temperature_flag", - "hourly_minimum_downward_shortwave_irradiance_at_earth_surface_quality_flag", - "hourly_minimum_soil_temperature_quality_flag", + "maximum_soil_temperature_flag", + "maximum_solar_irradiance_quality_flag", + "minimum_soil_temperature_quality_flag", + "minimum_solar_irradiance_quality_flag", "relative_humidity_quality_flag", "soil_temperature_quality_flag", + "solar_irradiance_quality_flag", ] assert not actual.var_has_quality_field("air_temperature") - assert actual.var_has_quality_field( - "downward_shortwave_irradiance_at_earth_surface" - ) + assert actual.var_has_quality_field("maximum_soil_temperature") assert actual.vars_with_quality_field == [ - "downward_shortwave_irradiance_at_earth_surface", - "hourly_maximum_downward_shortwave_irradiance_at_earth_surface", - "hourly_maximum_soil_temperature", - "hourly_minimum_downward_shortwave_irradiance_at_earth_surface", - "hourly_minimum_soil_temperature", + "maximum_soil_temperature", + "maximum_solar_irradiance", + "minimum_soil_temperature", + "minimum_solar_irradiance", "relative_humidity", "soil_temperature", + "solar_irradiance", ] assert ( - actual.get_var_quality_flag_field_name( - "downward_shortwave_irradiance_at_earth_surface" - ) - == "downward_shortwave_irradiance_at_earth_surface_quality_flag" + actual.get_var_quality_flag_field_name("maximum_soil_temperature") + == "maximum_soil_temperature_flag" ) assert actual.get_var_uncertainty_field_names("air_temperature") == [ "air_temperature_positive_total_uncertainty", diff --git a/tests/test_http_api.py b/tests/test_http_api.py index 1aa17be..64e0235 100644 --- a/tests/test_http_api.py +++ b/tests/test_http_api.py @@ -90,36 +90,65 @@ def test_get_dataset_auxiliary_variables_mapping(): "accumulated_precipitation": [], "air_temperature": [ { - "auxvar": "air_temperature_mean_positive_total_uncertainty", + "auxvar": "air_temperature_positive_total_uncertainty", "metadata_name": "positive_total_uncertainty", }, { - "auxvar": "air_temperature_mean_negative_total_uncertainty", + "auxvar": "air_temperature_negative_total_uncertainty", "metadata_name": "negative_total_uncertainty", }, + { + "auxvar": "air_temperature_positive_random_uncertainty", + "metadata_name": "positive_random_uncertainty", + }, + { + "auxvar": "air_temperature_negative_random_uncertainty", + "metadata_name": "negative_random_uncertainty", + }, + { + "auxvar": "air_temperature_positive_systematic_uncertainty", + "metadata_name": "positive_systematic_uncertainty", + }, + { + "auxvar": "air_temperature_negative_systematic_uncertainty", + "metadata_name": "negative_systematic_uncertainty", + }, ], + "daily_global_solar_radiation": [], "daily_maximum_air_temperature": [ { - "auxvar": "air_temperature_max_positive_total_uncertainty", + "auxvar": "daily_maximum_air_temperature_positive_total_uncertainty", "metadata_name": "positive_total_uncertainty", }, { - "auxvar": "air_temperature_max_negative_total_uncertainty", + "auxvar": "daily_maximum_air_temperature_negative_total_uncertainty", "metadata_name": "negative_total_uncertainty", }, ], "daily_maximum_relative_humidity": [], + "daily_mean_air_temperature": [ + { + "auxvar": "mean_air_temperature_positive_total_uncertainty", + "metadata_name": "positive_total_uncertainty", + }, + { + "auxvar": "mean_air_temperature_negative_total_uncertainty", + "metadata_name": "negative_total_uncertainty", + }, + ], "daily_minimum_air_temperature": [ { - "auxvar": "air_temperature_min_positive_total_uncertainty", + "auxvar": "daily_minimum_air_temperature_positive_total_uncertainty", "metadata_name": "positive_total_uncertainty", }, { - "auxvar": "air_temperature_min_negative_total_uncertainty", + "auxvar": "daily_minimum_air_temperature_negative_total_uncertainty", "metadata_name": "negative_total_uncertainty", }, ], "daily_minimum_relative_humidity": [], + "maximum_soil_temperature": [], + "minimum_soil_temperature": [], "relative_humidity": [], "soil_moisture_100cm_from_earth_surface": [], "soil_moisture_10cm_from_earth_surface": [], From 6147ab481726de312292ecebf67cb74fd2aec413 Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 13:25:18 +0200 Subject: [PATCH 07/20] fix copy --- cdsobs/cli/_copy_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdsobs/cli/_copy_dataset.py b/cdsobs/cli/_copy_dataset.py index 9c6961a..f560fb1 100644 --- a/cdsobs/cli/_copy_dataset.py +++ b/cdsobs/cli/_copy_dataset.py @@ -158,7 +158,7 @@ def copy_outside(init_config, dest_config, dataset, dest_dataset): with get_session(init_config.catalogue_db) as init_session: entries = CatalogueRepository(init_session).get_by_dataset(dataset) if init_config.s3config == dest_config.s3config: - new_assets = s3_copy(init_s3client, entries, dest_dataset) + new_assets = s3_copy(dest_config, entries, dest_dataset) dest_s3client = init_s3client else: # get new destination client as current client From 7173913db61a24b9316004ee03fc1da550b6a149 Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 13:26:36 +0200 Subject: [PATCH 08/20] fix copy 2 --- cdsobs/cli/_copy_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdsobs/cli/_copy_dataset.py b/cdsobs/cli/_copy_dataset.py index f560fb1..2f52bbe 100644 --- a/cdsobs/cli/_copy_dataset.py +++ b/cdsobs/cli/_copy_dataset.py @@ -158,7 +158,7 @@ def copy_outside(init_config, dest_config, dataset, dest_dataset): with get_session(init_config.catalogue_db) as init_session: entries = CatalogueRepository(init_session).get_by_dataset(dataset) if init_config.s3config == dest_config.s3config: - new_assets = s3_copy(dest_config, entries, dest_dataset) + new_assets = s3_copy(dest_config.s3config, entries, dest_dataset) dest_s3client = init_s3client else: # get new destination client as current client From 43fd9f1167c4fbd8a55e1a651e724b883675f98f Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 13:28:00 +0200 Subject: [PATCH 09/20] fix copy 3 --- cdsobs/cli/_copy_dataset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdsobs/cli/_copy_dataset.py b/cdsobs/cli/_copy_dataset.py index 2f52bbe..06e7090 100644 --- a/cdsobs/cli/_copy_dataset.py +++ b/cdsobs/cli/_copy_dataset.py @@ -158,7 +158,9 @@ def copy_outside(init_config, dest_config, dataset, dest_dataset): with get_session(init_config.catalogue_db) as init_session: entries = CatalogueRepository(init_session).get_by_dataset(dataset) if init_config.s3config == dest_config.s3config: - new_assets = s3_copy(dest_config.s3config, entries, dest_dataset) + # namespace may be different, so we need another 3 client here + dest_s3client = S3Client.from_config(init_config.s3config) + new_assets = s3_copy(dest_s3client, entries, dest_dataset) dest_s3client = init_s3client else: # get new destination client as current client From 1f6ab821b867d78f8728234a714cda931de1f659 Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 13:29:02 +0200 Subject: [PATCH 10/20] fix copy 4 --- cdsobs/cli/_copy_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdsobs/cli/_copy_dataset.py b/cdsobs/cli/_copy_dataset.py index 06e7090..9c667a0 100644 --- a/cdsobs/cli/_copy_dataset.py +++ b/cdsobs/cli/_copy_dataset.py @@ -159,7 +159,7 @@ def copy_outside(init_config, dest_config, dataset, dest_dataset): entries = CatalogueRepository(init_session).get_by_dataset(dataset) if init_config.s3config == dest_config.s3config: # namespace may be different, so we need another 3 client here - dest_s3client = S3Client.from_config(init_config.s3config) + dest_s3client = S3Client.from_config(dest_config.s3config) new_assets = s3_copy(dest_s3client, entries, dest_dataset) dest_s3client = init_s3client else: From 40cae1afd1f021e6087269797915ef9bf6964276 Mon Sep 17 00:00:00 2001 From: garciam Date: Fri, 26 Jul 2024 14:06:10 +0200 Subject: [PATCH 11/20] fix copy 5 --- cdsobs/cli/_copy_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cdsobs/cli/_copy_dataset.py b/cdsobs/cli/_copy_dataset.py index 9c667a0..59f7fdb 100644 --- a/cdsobs/cli/_copy_dataset.py +++ b/cdsobs/cli/_copy_dataset.py @@ -161,7 +161,6 @@ def copy_outside(init_config, dest_config, dataset, dest_dataset): # namespace may be different, so we need another 3 client here dest_s3client = S3Client.from_config(dest_config.s3config) new_assets = s3_copy(dest_s3client, entries, dest_dataset) - dest_s3client = init_s3client else: # get new destination client as current client dest_s3client = S3Client.from_config(dest_config.s3config) From 2bc9082798da8edc49fd4219dea79638e55f7d81 Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 29 Jul 2024 10:01:58 +0200 Subject: [PATCH 12/20] move config read function --- cdsobs/cli/_get_forms_jsons.py | 3 ++- cdsobs/cli/_make_cdm.py | 3 ++- cdsobs/cli/_make_production.py | 3 ++- cdsobs/cli/_utils.py | 14 -------------- cdsobs/config.py | 11 +++++++++++ tests/system/1_year_benchmarks.py | 2 +- tests/system/run_sanity_checks.py | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cdsobs/cli/_get_forms_jsons.py b/cdsobs/cli/_get_forms_jsons.py index 262ac71..22044a3 100644 --- a/cdsobs/cli/_get_forms_jsons.py +++ b/cdsobs/cli/_get_forms_jsons.py @@ -3,7 +3,8 @@ import typer -from cdsobs.cli._utils import config_yml_typer, read_and_validate_config +from cdsobs.cli._utils import config_yml_typer +from cdsobs.config import read_and_validate_config from cdsobs.forms_jsons import get_forms_jsons from cdsobs.observation_catalogue.database import get_session from cdsobs.observation_catalogue.repositories.catalogue import CatalogueRepository diff --git a/cdsobs/cli/_make_cdm.py b/cdsobs/cli/_make_cdm.py index 6adaa2a..b899020 100644 --- a/cdsobs/cli/_make_cdm.py +++ b/cdsobs/cli/_make_cdm.py @@ -4,7 +4,8 @@ import typer from cdsobs.api import run_make_cdm -from cdsobs.cli._utils import config_yml_typer, read_and_validate_config +from cdsobs.cli._utils import config_yml_typer +from cdsobs.config import read_and_validate_config from cdsobs.service_definition.api import validate_service_definition diff --git a/cdsobs/cli/_make_production.py b/cdsobs/cli/_make_production.py index 924c586..25fa7ca 100644 --- a/cdsobs/cli/_make_production.py +++ b/cdsobs/cli/_make_production.py @@ -3,7 +3,8 @@ import typer from cdsobs.api import run_ingestion_pipeline -from cdsobs.cli._utils import config_yml_typer, read_and_validate_config +from cdsobs.cli._utils import config_yml_typer +from cdsobs.config import read_and_validate_config from cdsobs.observation_catalogue.database import get_session from cdsobs.service_definition.api import validate_service_definition diff --git a/cdsobs/cli/_utils.py b/cdsobs/cli/_utils.py index b2c4e4b..503d8c3 100644 --- a/cdsobs/cli/_utils.py +++ b/cdsobs/cli/_utils.py @@ -1,12 +1,8 @@ -from pathlib import Path - import typer from botocore.exceptions import BotoCoreError from rich.console import Console from sqlalchemy.exc import OperationalError -from cdsobs.config import CDSObsConfig, validate_config - class CliException(Exception): pass @@ -60,13 +56,3 @@ def list_parser(arg: str): return [] else: return arg.split(",") - - -def read_and_validate_config(cdsobs_config_yml: Path | None) -> CDSObsConfig: - # read and validate config yaml - if cdsobs_config_yml is None: - cdsobs_config_yml = Path.home().joinpath(".cdsobs/cdsobs_config.yml") - if not Path(cdsobs_config_yml).exists(): - raise ConfigNotFound() - config = validate_config(cdsobs_config_yml) - return config diff --git a/cdsobs/config.py b/cdsobs/config.py index 54313c5..a2ab435 100644 --- a/cdsobs/config.py +++ b/cdsobs/config.py @@ -7,6 +7,7 @@ import yaml from pydantic_core.core_schema import ValidationInfo +from cdsobs.cli._utils import ConfigNotFound from cdsobs.utils.exceptions import ConfigError from cdsobs.utils.types import LatTileSize, LonTileSize, TimeTileSize @@ -220,3 +221,13 @@ def validate_config(config_file: Path): with config_file.open() as f: config_dict = yaml.safe_load(f) return CDSObsConfig(**config_dict) + + +def read_and_validate_config(cdsobs_config_yml: Path | None) -> CDSObsConfig: + # read and validate config yaml + if cdsobs_config_yml is None: + cdsobs_config_yml = Path.home().joinpath(".cdsobs/cdsobs_config.yml") + if not Path(cdsobs_config_yml).exists(): + raise ConfigNotFound() + config = validate_config(cdsobs_config_yml) + return config diff --git a/tests/system/1_year_benchmarks.py b/tests/system/1_year_benchmarks.py index e21a7a8..6c42f3c 100644 --- a/tests/system/1_year_benchmarks.py +++ b/tests/system/1_year_benchmarks.py @@ -12,7 +12,7 @@ from cdsobs.api import run_ingestion_pipeline from cdsobs.cli._delete_dataset import delete_from_catalogue -from cdsobs.cli._utils import read_and_validate_config +from cdsobs.config import read_and_validate_config from cdsobs.constants import TEST_DATASETS, TEST_YEARS, VARS2DATASET from cdsobs.observation_catalogue.database import get_session from cdsobs.observation_catalogue.models import Catalogue diff --git a/tests/system/run_sanity_checks.py b/tests/system/run_sanity_checks.py index d090e7c..4a648c0 100644 --- a/tests/system/run_sanity_checks.py +++ b/tests/system/run_sanity_checks.py @@ -1,7 +1,7 @@ import os from pathlib import Path -from cdsobs.cli._utils import read_and_validate_config +from cdsobs.config import read_and_validate_config from cdsobs.constants import TEST_DATASETS, TEST_YEARS from cdsobs.sanity_checks import run_sanity_checks from cdsobs.utils.logutils import get_logger From dd2f69d86e25085804288efd7d2d5ff9c2ec3164 Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 29 Jul 2024 10:06:28 +0200 Subject: [PATCH 13/20] move CLI exceptions to exception module --- cdsobs/api_rest/endpoints.py | 3 +-- cdsobs/cli/_catalogue_explorer.py | 4 +--- cdsobs/cli/_copy_dataset.py | 4 ++-- cdsobs/cli/_delete_dataset.py | 4 +--- cdsobs/cli/_retrieve.py | 3 ++- cdsobs/cli/_utils.py | 10 +--------- cdsobs/cli/_validate.py | 2 +- cdsobs/config.py | 3 +-- cdsobs/utils/exceptions.py | 10 ++++++++++ 9 files changed, 20 insertions(+), 23 deletions(-) diff --git a/cdsobs/api_rest/endpoints.py b/cdsobs/api_rest/endpoints.py index 6bc36e3..d27edf0 100644 --- a/cdsobs/api_rest/endpoints.py +++ b/cdsobs/api_rest/endpoints.py @@ -9,7 +9,6 @@ from cdsobs.api_rest.models import RetrievePayload from cdsobs.cdm.api import get_auxiliary_variables_mapping from cdsobs.cdm.lite import cdm_lite_variables -from cdsobs.cli._utils import ConfigNotFound from cdsobs.config import CDSObsConfig, validate_config from cdsobs.ingestion.core import get_variables_from_service_definition from cdsobs.observation_catalogue.repositories.cads_dataset import CadsDatasetRepository @@ -21,7 +20,7 @@ from cdsobs.service_definition.api import get_service_definition from cdsobs.service_definition.service_definition_models import ServiceDefinition from cdsobs.storage import S3Client -from cdsobs.utils.exceptions import DataNotFoundException, SizeError +from cdsobs.utils.exceptions import ConfigNotFound, DataNotFoundException, SizeError from cdsobs.utils.utils import get_database_session router = APIRouter() diff --git a/cdsobs/cli/_catalogue_explorer.py b/cdsobs/cli/_catalogue_explorer.py index 5176fc3..a50cc81 100644 --- a/cdsobs/cli/_catalogue_explorer.py +++ b/cdsobs/cli/_catalogue_explorer.py @@ -18,11 +18,9 @@ from cdsobs.observation_catalogue.schemas.catalogue import CliCatalogueFilters from cdsobs.utils.logutils import get_logger -from ..utils.exceptions import ConfigError +from ..utils.exceptions import CliException, ConfigError, ConfigNotFound from ._utils import ( PAGE_SIZE, - CliException, - ConfigNotFound, config_yml_typer, list_parser, print_format_msg, diff --git a/cdsobs/cli/_copy_dataset.py b/cdsobs/cli/_copy_dataset.py index 59f7fdb..77f964c 100644 --- a/cdsobs/cli/_copy_dataset.py +++ b/cdsobs/cli/_copy_dataset.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import Session from typer import Option -from cdsobs.cli._utils import CliException, ConfigNotFound, config_yml_typer +from cdsobs.cli._utils import config_yml_typer from cdsobs.config import CDSObsConfig from cdsobs.observation_catalogue.database import get_session from cdsobs.observation_catalogue.models import Catalogue @@ -17,7 +17,7 @@ from cdsobs.observation_catalogue.repositories.catalogue import CatalogueRepository from cdsobs.observation_catalogue.schemas.catalogue import CatalogueSchema from cdsobs.storage import S3Client -from cdsobs.utils.exceptions import ConfigError +from cdsobs.utils.exceptions import CliException, ConfigError, ConfigNotFound from cdsobs.utils.logutils import get_logger logger = get_logger(__name__) diff --git a/cdsobs/cli/_delete_dataset.py b/cdsobs/cli/_delete_dataset.py index cb85cb4..2887208 100644 --- a/cdsobs/cli/_delete_dataset.py +++ b/cdsobs/cli/_delete_dataset.py @@ -7,8 +7,6 @@ from rich.console import Console from cdsobs.cli._utils import ( - CliException, - ConfigNotFound, config_yml_typer, list_parser, ) @@ -20,7 +18,7 @@ CliCatalogueFilters, ) from cdsobs.storage import S3Client -from cdsobs.utils.exceptions import ConfigError +from cdsobs.utils.exceptions import CliException, ConfigError, ConfigNotFound console = Console() diff --git a/cdsobs/cli/_retrieve.py b/cdsobs/cli/_retrieve.py index cf0ea8f..c6d0597 100644 --- a/cdsobs/cli/_retrieve.py +++ b/cdsobs/cli/_retrieve.py @@ -5,11 +5,12 @@ import typer from rich.console import Console -from cdsobs.cli._utils import CliException, ConfigNotFound, config_yml_typer +from cdsobs.cli._utils import config_yml_typer from cdsobs.config import validate_config from cdsobs.retrieve.api import retrieve_observations from cdsobs.retrieve.models import RetrieveArgs from cdsobs.storage import S3Client +from cdsobs.utils.exceptions import CliException, ConfigNotFound def retrieve( diff --git a/cdsobs/cli/_utils.py b/cdsobs/cli/_utils.py index 503d8c3..41cb0ab 100644 --- a/cdsobs/cli/_utils.py +++ b/cdsobs/cli/_utils.py @@ -3,15 +3,7 @@ from rich.console import Console from sqlalchemy.exc import OperationalError - -class CliException(Exception): - pass - - -class ConfigNotFound(CliException): - def __init__(self, msg="Configuration yaml not found"): - self.message = msg - super().__init__(self.message) +from cdsobs.utils.exceptions import CliException def exception_handler(exception): diff --git a/cdsobs/cli/_validate.py b/cdsobs/cli/_validate.py index 234e8c2..4611081 100644 --- a/cdsobs/cli/_validate.py +++ b/cdsobs/cli/_validate.py @@ -3,7 +3,7 @@ import typer import cdsobs.service_definition.api -from cdsobs.cli._utils import CliException +from cdsobs.utils.exceptions import CliException def validate_service_definition( diff --git a/cdsobs/config.py b/cdsobs/config.py index a2ab435..1dd49c7 100644 --- a/cdsobs/config.py +++ b/cdsobs/config.py @@ -7,8 +7,7 @@ import yaml from pydantic_core.core_schema import ValidationInfo -from cdsobs.cli._utils import ConfigNotFound -from cdsobs.utils.exceptions import ConfigError +from cdsobs.utils.exceptions import ConfigError, ConfigNotFound from cdsobs.utils.types import LatTileSize, LonTileSize, TimeTileSize diff --git a/cdsobs/utils/exceptions.py b/cdsobs/utils/exceptions.py index 24aabcf..22e51e8 100644 --- a/cdsobs/utils/exceptions.py +++ b/cdsobs/utils/exceptions.py @@ -12,3 +12,13 @@ class CatalogueException(Exception): class DataNotFoundException(RuntimeError): pass + + +class CliException(Exception): + pass + + +class ConfigNotFound(CliException): + def __init__(self, msg="Configuration yaml not found"): + self.message = msg + super().__init__(self.message) From 1fd2037ee780fa934c4a14f72f8ff1e98f909cb8 Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 29 Jul 2024 11:00:42 +0200 Subject: [PATCH 14/20] cli for storage checks uses now the logger not ruch console --- cdsobs/cli/_object_storage.py | 22 ++++++++++------------ cdsobs/cli/_utils.py | 15 +++++++-------- tests/cli/test_object_storage.py | 20 ++++++++++++-------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/cdsobs/cli/_object_storage.py b/cdsobs/cli/_object_storage.py index 836c3bd..a97899d 100644 --- a/cdsobs/cli/_object_storage.py +++ b/cdsobs/cli/_object_storage.py @@ -1,15 +1,15 @@ from pathlib import Path import typer -from rich.console import Console from cdsobs.cli._utils import config_yml_typer from cdsobs.config import CDSObsConfig from cdsobs.observation_catalogue.database import get_session from cdsobs.observation_catalogue.repositories.catalogue import CatalogueRepository from cdsobs.storage import S3Client, StorageClient +from cdsobs.utils.logutils import get_logger -console = Console() +logger = get_logger(__name__) def check_consistency( @@ -38,14 +38,14 @@ def check_if_missing_in_object_storage( ): red_flag = False if dataset is None: - console.print( - "Check if every dataset in the catalogue is in the object storage" + logger.info( + "Checking if every dataset in the catalogue is in the object storage" ) page = 0 page_size = 10000 assets = catalogue_repo.get_all_assets(limit=page_size) while len(assets): - console.print(f"Checking page {page+1} ({page_size} records per page)") + logger.info(f"Checking page {page+1} ({page_size} records per page)") red_flag = assets_in_s3(assets, s3client) page += 1 assets = catalogue_repo.get_all_assets( @@ -55,7 +55,7 @@ def check_if_missing_in_object_storage( assets = catalogue_repo.get_dataset_assets(dataset) red_flag = assets_in_s3(assets, s3client) if not red_flag: - console.print("[bold green] Found all assets in object storage [/bold green]") + logger.info("Found all assets in object storage.") def assets_in_s3(assets, s3client) -> bool: @@ -63,9 +63,7 @@ def assets_in_s3(assets, s3client) -> bool: for asset in assets: bucket, name = asset.split("/") if not s3client.object_exists(bucket, name): - console.print( - f"[bold red] Missing {str(asset)} in object storage [/bold red]" - ) + logger.warning(f"Missing {str(asset)} in object storage.") red_flag = True return red_flag @@ -77,7 +75,7 @@ def check_if_missing_in_catalogue( ): red_flag = False if dataset is None: - console.print( + logger.info( "Check if every dataset in the object storage has a catalogue entry" ) buckets = s3client.list_buckets() @@ -91,7 +89,7 @@ def check_if_missing_in_catalogue( objects = s3client.list_directory_objects(bucket) red_flag = objects_in_catalogue(objects, bucket, s3client, catalogue_repo) if not red_flag: - console.print("[bold green] Found all assets in catalogue [/bold green]") + logger.info("Found all assets in catalogue.") def objects_in_catalogue(objects, bucket, s3client, catalogue_repo) -> bool: @@ -99,6 +97,6 @@ def objects_in_catalogue(objects, bucket, s3client, catalogue_repo) -> bool: for obj in objects: asset = s3client.get_asset(bucket, obj) if not catalogue_repo.exists_asset(asset): - console.print(f"[bold red] Missing {asset} entry in catalogue [/bold red]") + logger.warning(f"Missing {asset} entry in catalogue.") red_flag = True return red_flag diff --git a/cdsobs/cli/_utils.py b/cdsobs/cli/_utils.py index 41cb0ab..53eaa1f 100644 --- a/cdsobs/cli/_utils.py +++ b/cdsobs/cli/_utils.py @@ -1,28 +1,27 @@ import typer from botocore.exceptions import BotoCoreError -from rich.console import Console from sqlalchemy.exc import OperationalError from cdsobs.utils.exceptions import CliException +from cdsobs.utils.logutils import get_logger + +logger = get_logger(__name__) def exception_handler(exception): more_info = "Set CLI_DEBUG=true to see the full trace" - err_console = Console(stderr=True) if isinstance(exception, CliException): - err_console.print(f"Error:\\ {exception}") + logger.error(f"Error:\\ {exception}") elif isinstance(exception, BotoCoreError): - err_console.print( + logger.error( f"Error connecting or operating with the storage. {more_info}:\n {exception}." ) elif isinstance(exception, OperationalError): - err_console.print( + logger.error( f"Error connecting or querying the database. {more_info}:\n {exception}." ) else: - err_console.print( - f"Error: ({type(exception).__name__}: {exception}), {more_info}" - ) + logger.error(f"Error: ({type(exception).__name__}: {exception}), {more_info}") config_yml_typer = typer.Option( diff --git a/tests/cli/test_object_storage.py b/tests/cli/test_object_storage.py index ddd05e9..7800815 100644 --- a/tests/cli/test_object_storage.py +++ b/tests/cli/test_object_storage.py @@ -1,4 +1,5 @@ import pytest_mock +from structlog.testing import capture_logs from typer.testing import CliRunner from cdsobs.cli._object_storage import ( @@ -16,13 +17,14 @@ def test_check_if_missing_in_storage( test_session, test_s3_client, test_repository, - capsys, ): catalogue_repo = CatalogueRepository(test_session) mocker.patch.object(test_s3_client, "object_exists", return_value=True) - check_if_missing_in_object_storage(catalogue_repo, test_s3_client, DS_TEST_NAME) - captured_out = capsys.readouterr().out - assert "Found all assets in object storage" in captured_out + with capture_logs() as cap_logs: + check_if_missing_in_object_storage(catalogue_repo, test_s3_client, DS_TEST_NAME) + assert cap_logs == [ + {"event": "Found all assets in object storage.", "log_level": "info"} + ] def test_check_if_missing_in_catalogue( @@ -36,7 +38,9 @@ def test_check_if_missing_in_catalogue( mocker.patch.object( test_s3_client, "list_directory_objects", side_effect=[["test_object"], []] ) - check_if_missing_in_catalogue(catalogue_repo, test_s3_client) - captured_out = capsys.readouterr().out - assert "Missing" in captured_out - assert "test_bucket/test_object entry in catalogue" in captured_out + with capture_logs() as cap_logs: + check_if_missing_in_catalogue(catalogue_repo, test_s3_client) + assert cap_logs[-1] == { + "event": "Missing test_bucket/test_object entry in catalogue.", + "log_level": "warning", + } From c74548df3b7c8cfe58fb6a8dac2386d8149299ae Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 29 Jul 2024 14:48:05 +0200 Subject: [PATCH 15/20] total uncertainty -> combined_uncertainty --- .../service_definition.yml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cdsobs/data/insitu-observations-gnss/service_definition.yml b/cdsobs/data/insitu-observations-gnss/service_definition.yml index d365749..701d5ba 100644 --- a/cdsobs/data/insitu-observations-gnss/service_definition.yml +++ b/cdsobs/data/insitu-observations-gnss/service_definition.yml @@ -86,7 +86,7 @@ sources: dtype: object long_name: agency precipitable_water_column: - total_uncertainty: uncert_epn_ipw + combined_uncertainty: uncert_epn_ipw description: This parameter is the total amount of water vapour in a column extending vertically from the GNSS receiver position (near the surface) to the top of the atmosphere. dtype: float32 era5: era_ipw @@ -112,10 +112,10 @@ sources: description: This parameter indicates the name of the GNSS receiving station. dtype: object long_name: idstation - precipitable_water_column_total_uncertainty: + precipitable_water_column_combined_uncertainty: description: This parameter is the combined sum of all uncertainties in the total column water vapour derived from zenith total delay and ancillary meteorological data. The uncertainties that are included in the calculation include uncertainties of the observed zenith total delay, uncertainties of the ancillary data, and uncertainties of the coefficients used in the retrieval (csv-lev only). dtype: float32 - long_name: precipitable_water_column_total_uncertainty + long_name: precipitable_water_column_combined_uncertainty units: kg m-2 precipitable_water_column_era5: description: This parameter is the total amount of water vapour in a column extending vertically from the GNSS receiver position (near the surface) to the top of the atmosphere, retrieved from ERA5 at the station coordinates, altitude, date, and time (csv-lev only). @@ -168,7 +168,7 @@ sources: group_name: variables - columns: - uncert_epn_ipw - group_name: total_uncertainty + group_name: combined_uncertainty - columns: - sigma_ztd group_name: random_uncertainty @@ -195,7 +195,7 @@ sources: network: network observation_id: observation_id sigma_ztd: zenith_total_delay_random_uncertainty - uncert_gnss_ipw: precipitable_water_column_total_uncertainty + uncert_gnss_ipw: precipitable_water_column_combined_uncertainty ztd: zenith_total_delay unit_changes: precipitable_water_column: @@ -237,7 +237,7 @@ sources: dtype: object long_name: agency precipitable_water_column: - total_uncertainty: uncert_gnss_ipw + combined_uncertainty: uncert_gnss_ipw description: This parameter is the total amount of water vapour in a column extending vertically from the GNSS receiver position (near the surface) to the top of the atmosphere. dtype: float32 era5: era_ipw @@ -264,10 +264,10 @@ sources: dtype: object long_name: idstation name_for_output: station_name - precipitable_water_column_total_uncertainty: + precipitable_water_column_combined_uncertainty: description: This parameter is the combined sum of all uncertainties in the total column water vapour derived from zenith total delay and ancillary meteorological data. The uncertainties that are included in the calculation include uncertainties of the observed zenith total delay, uncertainties of the ancillary data, and uncertainties of the coefficients used in the retrieval (csv-lev only). dtype: float32 - long_name: precipitable_water_column_total_uncertainty + long_name: precipitable_water_column_combined_uncertainty units: kg m-2 precipitable_water_column_era5: description: This parameter is the total amount of water vapour in a column extending vertically from the GNSS receiver position (near the surface) to the top of the atmosphere, retrieved from ERA5 at the station coordinates, altitude, date, and time (csv-lev only). @@ -320,7 +320,7 @@ sources: group_name: variables - columns: - uncert_gnss_ipw - group_name: total_uncertainty + group_name: combined_uncertainty - columns: - sigma_ztd group_name: random_uncertainty @@ -347,7 +347,7 @@ sources: network: network observation_id: observation_id sigma_ztd: zenith_total_delay_random_uncertainty - uncert_gnss_ipw: precipitable_water_column_total_uncertainty + uncert_gnss_ipw: precipitable_water_column_combined_uncertainty ztd: zenith_total_delay unit_changes: precipitable_water_column: @@ -389,7 +389,7 @@ sources: dtype: object long_name: agency precipitable_water_column: - total_uncertainty: uncert_gnss_ipw + combined_uncertainty: uncert_gnss_ipw description: This parameter is the total amount of water vapour in a column extending vertically from the GNSS receiver position (near the surface) to the top of the atmosphere. dtype: float32 era5: era_ipw @@ -416,10 +416,10 @@ sources: dtype: object long_name: idstation name_for_output: station_name - precipitable_water_column_total_uncertainty: + precipitable_water_column_combined_uncertainty: description: This parameter is the combined sum of all uncertainties in the total column water vapour derived from zenith total delay and ancillary meteorological data. The uncertainties that are included in the calculation include uncertainties of the observed zenith total delay, uncertainties of the ancillary data, and uncertainties of the coefficients used in the retrieval (csv-lev only). dtype: float32 - long_name: precipitable_water_column_total_uncertainty + long_name: precipitable_water_column_combined_uncertainty units: kg m-2 precipitable_water_column_era5: description: This parameter is the total amount of water vapour in a column extending vertically from the GNSS receiver position (near the surface) to the top of the atmosphere, retrieved from ERA5 at the station coordinates, altitude, date, and time (csv-lev only). @@ -472,7 +472,7 @@ sources: group_name: variables - columns: - uncert_gnss_ipw - group_name: total_uncertainty + group_name: combined_uncertainty - columns: - sigma_ztd group_name: random_uncertainty From 08c4c1f51aa6be143f912d0c067ab4f2cf711b83 Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 29 Jul 2024 14:53:58 +0200 Subject: [PATCH 16/20] bump python to 3.12 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 3314e96..2728283 100644 --- a/environment.yml +++ b/environment.yml @@ -44,7 +44,7 @@ dependencies: - pydantic-settings - datetimerange - uvicorn -- python=3.10 +- python=3.12 - pip: - sqlalchemy_json - "git+https://github.com/ecmwf-projects/cads-adaptors" From ab3d6c3cdf379e39b94dd4ee07717df3026e722a Mon Sep 17 00:00:00 2001 From: garciam Date: Wed, 31 Jul 2024 09:32:19 +0200 Subject: [PATCH 17/20] added combined_uncertainty --- cdsobs/cdm/lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdsobs/cdm/lite.py b/cdsobs/cdm/lite.py index dbb6628..d96e4ff 100644 --- a/cdsobs/cdm/lite.py +++ b/cdsobs/cdm/lite.py @@ -74,6 +74,7 @@ "positive_quasisystematic_uncertainty", "negative_quasisystematic_uncertainty", "quality_flag", + "combined_uncertainty", ] cdm_lite_variables = dict( mandatory=variable_names, From ad66565c51e641eb4f01e6012d02ef3bd204bf77 Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 5 Aug 2024 14:38:14 +0200 Subject: [PATCH 18/20] added support for processing level --- cdsobs/cdm/api.py | 22 +++++ cdsobs/cdm/lite.py | 1 + .../service_definition.yml | 2 +- cdsobs/ingestion/api.py | 17 +++- cdsobs/ingestion/serialize.py | 3 + tests/test_api.py | 97 +++++++++++++------ 6 files changed, 110 insertions(+), 32 deletions(-) diff --git a/cdsobs/cdm/api.py b/cdsobs/cdm/api.py index 8128a82..ea59361 100644 --- a/cdsobs/cdm/api.py +++ b/cdsobs/cdm/api.py @@ -83,6 +83,13 @@ def to_cdm_dataset(partition: DatasetPartition) -> CdmDataset: cdm_variables += cdm_variables_with_table_names cdm_variables = unique([v for v in cdm_variables if v in partition.data]) data = partition.data.loc[:, cdm_variables].set_index("observation_id") + original_variables = set(partition.data.columns) + removed_variables = original_variables - set(cdm_variables) + if len(removed_variables) > 0: + logger.warning( + "The following variables where read but are not in the CDM and " + f"are going to be dropped: {removed_variables}" + ) return CdmDataset(data, partition.partition_params, partition.dataset_metadata) @@ -330,6 +337,21 @@ def auxfield2metadata_name(self, var: str, aux_var: str) -> str: auxf["metadata_name"] for auxf in self[var] if auxf["auxvar"] == aux_var ][0] + def vars_with_processing_level(self) -> list[str]: + return [v for v in self if self.var_has_processing_level(v)] + + def var_has_processing_level(self, var: str) -> bool: + return any(auxf["auxvar"] in self.processing_level_fields for auxf in self[var]) + + @property + def processing_level_fields(self) -> list[str]: + return [auxf for auxf in self.all_list if "processing_level" in auxf] + + def get_var_processing_level_field_name(self, var: str) -> str: + return [ + auxf["auxvar"] for auxf in self[var] if "processing_level" in auxf["auxvar"] + ][0] + def get_aux_fields_mapping_from_service_definition( source_definition: SourceDefinition, variables: List[str] diff --git a/cdsobs/cdm/lite.py b/cdsobs/cdm/lite.py index d96e4ff..14f1a73 100644 --- a/cdsobs/cdm/lite.py +++ b/cdsobs/cdm/lite.py @@ -75,6 +75,7 @@ "negative_quasisystematic_uncertainty", "quality_flag", "combined_uncertainty", + "processing_level", ] cdm_lite_variables = dict( mandatory=variable_names, diff --git a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml index 1584ab4..7884f2a 100644 --- a/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml +++ b/cdsobs/data/insitu-observations-near-surface-temperature-us-climate-reference-network/service_definition.yml @@ -722,7 +722,7 @@ sources: units: K soil_temperature_processing_level: description: 'This parameter indicates the level of processing applied to the soil temperature measurement: raw data (''R''), corrected data (''C''). A letter ''U'' indicates that this information is unknown.' - dtype: float32 + dtype: object long_name: sur_temp_type soil_temperature_quality_flag: description: This parameter indicates if the "Soil temperature" (described in this list) may be used because it is based on good data (value of 0) or if it should be treated with suspicion because erroneous data were detected (value of 3). diff --git a/cdsobs/ingestion/api.py b/cdsobs/ingestion/api.py index efbd43c..46c0e7e 100644 --- a/cdsobs/ingestion/api.py +++ b/cdsobs/ingestion/api.py @@ -317,8 +317,9 @@ def _melt_variables( ) # Add quality flags - homogenised_data_melted["quality_flag"] = 3.0 vars_with_qf = aux_fields.vars_with_quality_field + if len(vars_with_qf) > 0: + homogenised_data_melted["quality_flag"] = 3.0 for var in vars_with_qf: var_mask = homogenised_data_melted["observed_variable"] == var flag_name = aux_fields.get_var_quality_flag_field_name(var) @@ -330,6 +331,20 @@ def _melt_variables( homogenised_data_melted["quality_flag"] = ( homogenised_data_melted["quality_flag"].fillna(3).astype("int") ) + # Add processing level + vars_with_pl = aux_fields.vars_with_processing_level() + if len(vars_with_pl) > 0: + homogenised_data_melted["processing_level"] = 6 + for var in vars_with_pl: + var_mask = homogenised_data_melted["observed_variable"] == var + pl_name = aux_fields.get_var_processing_level_field_name(var) + var_processing_level = homogenised_data_melted.loc[var_mask, pl_name] + homogenised_data_melted.loc[ + var_mask, "processing_level" + ] = var_processing_level + homogenised_data_melted = homogenised_data_melted.drop(pl_name, axis=1) + + homogenised_data_melted["processing_level"] = 6 # Encode observed_variables logger.info("Encoding observed variables using the CDM variable codes.") code_table = read_cdm_code_table(cdm_tables_location, "observed_variable").table diff --git a/cdsobs/ingestion/serialize.py b/cdsobs/ingestion/serialize.py index 3bc2a55..8aa33cb 100644 --- a/cdsobs/ingestion/serialize.py +++ b/cdsobs/ingestion/serialize.py @@ -106,6 +106,9 @@ def write_pandas_to_netcdf( if attrs is not None and v in attrs: ovar.attrs.update(attrs[v]) + oncobj.sync() + oncobj.close() + def to_netcdf( cdm_dataset: CdmDataset, tempdir: Path, encode_variables: bool = True diff --git a/tests/test_api.py b/tests/test_api.py index 6f70535..2af9e61 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -5,28 +5,35 @@ import sqlalchemy as sa from cdsobs.api import run_ingestion_pipeline, run_make_cdm +from cdsobs.cdm.api import open_netcdf +from cdsobs.cdm.lite import auxiliary_variable_names +from cdsobs.ingestion.core import get_aux_vars_from_service_definition from cdsobs.observation_catalogue.models import Catalogue from cdsobs.service_definition.api import get_service_definition +from cdsobs.storage import S3Client +from cdsobs.utils.logutils import get_logger from tests.utils import get_test_years +logger = get_logger(__name__) + @pytest.mark.parametrize( "dataset_name,source,test_update", [ - ( - "insitu-observations-woudc-ozone-total-column-and-profiles", - "OzoneSonde", - False, - ), - ( - "insitu-observations-woudc-ozone-total-column-and-profiles", - "TotalOzone", - False, - ), - ("insitu-observations-igra-baseline-network", "IGRA", False), - ("insitu-observations-igra-baseline-network", "IGRA_H", False), - ("insitu-comprehensive-upper-air-observation-network", "CUON", True), - ("insitu-observations-gruan-reference-network", "GRUAN", False), + # ( + # "insitu-observations-woudc-ozone-total-column-and-profiles", + # "OzoneSonde", + # False, + # ), + # ( + # "insitu-observations-woudc-ozone-total-column-and-profiles", + # "TotalOzone", + # False, + # ), + # ("insitu-observations-igra-baseline-network", "IGRA", False), + # ("insitu-observations-igra-baseline-network", "IGRA_H", False), + # ("insitu-comprehensive-upper-air-observation-network", "CUON", True), + # ("insitu-observations-gruan-reference-network", "GRUAN", False), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", "uscrn_subhourly", @@ -47,25 +54,25 @@ "uscrn_monthly", False, ), - ( - "insitu-observations-gnss", - "IGS", - False, - ), - ( - "insitu-observations-gnss", - "EPN", - False, - ), - ( - "insitu-observations-gnss", - "IGS_R3", - False, - ), + # ( + # "insitu-observations-gnss", + # "IGS", + # False, + # ), + # ( + # "insitu-observations-gnss", + # "EPN", + # False, + # ), + # ( + # "insitu-observations-gnss", + # "IGS_R3", + # False, + # ), ], ) def test_run_ingestion_pipeline( - dataset_name, source, test_update, test_session, test_config, caplog + dataset_name, source, test_update, test_session, test_config, caplog, tmp_path ): start_year, end_year = get_test_years(source) service_definition = get_service_definition(dataset_name) @@ -83,6 +90,36 @@ def test_run_ingestion_pipeline( # assert insertions have been made counter = test_session.scalar(sa.select(sa.func.count()).select_from(Catalogue)) assert counter > 0 + # Check variables + asset = test_session.scalar(sa.select(Catalogue.asset)) + s3client = S3Client.from_config(test_config.s3config) + asset_filename = asset.split("/")[1] + asset_local_path = Path(tmp_path, asset_filename) + s3client.download_file( + s3client.get_bucket_name(dataset_name), asset_filename, asset_local_path + ) + dataset = open_netcdf(asset_local_path, decode_variables=True) + variables_in_file = set( + dataset.columns.tolist() + dataset.observed_variable.unique().tolist() + ) + aux_variables = get_aux_vars_from_service_definition(service_definition, source) + expected_variables = set(service_definition.sources[source].descriptions) - set( + aux_variables + ) + for v in [ + "observed_variable", + "observation_value", + "units", + ] + auxiliary_variable_names: + if v in variables_in_file: + expected_variables.add(v) + logger.info( + f"{variables_in_file - expected_variables} are in file but not in the descriptions" + ) + logger.info( + f"{expected_variables - variables_in_file} are not in file but are in the description" + ) + # assert variables_in_file == expected_variables if test_update: # testing update flag From f87fd6591fc27933cbe05e33ef79ff09a9e78754 Mon Sep 17 00:00:00 2001 From: garciam Date: Mon, 5 Aug 2024 18:52:00 +0200 Subject: [PATCH 19/20] tests fixed --- cdsobs/cdm/api.py | 5 ++- cdsobs/ingestion/api.py | 10 +++--- cdsobs/utils/utils.py | 13 ++++++-- tests/test_api.py | 68 ++++++++++++++++++++++------------------- tests/test_cdm_api.py | 11 ++++++- tests/test_http_api.py | 7 ++++- 6 files changed, 72 insertions(+), 42 deletions(-) diff --git a/cdsobs/cdm/api.py b/cdsobs/cdm/api.py index ea59361..d0dbb84 100644 --- a/cdsobs/cdm/api.py +++ b/cdsobs/cdm/api.py @@ -349,7 +349,10 @@ def processing_level_fields(self) -> list[str]: def get_var_processing_level_field_name(self, var: str) -> str: return [ - auxf["auxvar"] for auxf in self[var] if "processing_level" in auxf["auxvar"] + auxf["auxvar"] + for auxf in self[var] + if "processing_level" in auxf["auxvar"] + and "quality_flag" not in auxf["auxvar"] ][0] diff --git a/cdsobs/ingestion/api.py b/cdsobs/ingestion/api.py index 46c0e7e..b49655a 100644 --- a/cdsobs/ingestion/api.py +++ b/cdsobs/ingestion/api.py @@ -326,11 +326,10 @@ def _melt_variables( var_quality_flag = homogenised_data_melted.loc[var_mask, flag_name] homogenised_data_melted.loc[var_mask, "quality_flag"] = var_quality_flag homogenised_data_melted = homogenised_data_melted.drop(flag_name, axis=1) - - # Ensure is int and fill nans with 3 (missing according to the CDM) - homogenised_data_melted["quality_flag"] = ( - homogenised_data_melted["quality_flag"].fillna(3).astype("int") - ) + # Ensure is int and fill nans with 3 (missing according to the CDM) + homogenised_data_melted["quality_flag"] = ( + homogenised_data_melted["quality_flag"].fillna(3).astype("int") + ) # Add processing level vars_with_pl = aux_fields.vars_with_processing_level() if len(vars_with_pl) > 0: @@ -344,7 +343,6 @@ def _melt_variables( ] = var_processing_level homogenised_data_melted = homogenised_data_melted.drop(pl_name, axis=1) - homogenised_data_melted["processing_level"] = 6 # Encode observed_variables logger.info("Encoding observed variables using the CDM variable codes.") code_table = read_cdm_code_table(cdm_tables_location, "observed_variable").table diff --git a/cdsobs/utils/utils.py b/cdsobs/utils/utils.py index cdaf7af..b201c2e 100644 --- a/cdsobs/utils/utils.py +++ b/cdsobs/utils/utils.py @@ -62,10 +62,19 @@ def get_code_mapping( attrs = incobj["observed_variable"].attrs else: raise RuntimeError("Unsupported input type") + labels = attrs["labels"] + codes = attrs["codes"] + if not isinstance(attrs["labels"], list): + labels = [ + labels, + ] + codes = [ + codes, + ] if inverse: - mapping = {c: v for v, c in zip(attrs["labels"], attrs["codes"])} + mapping = {c: v for v, c in zip(labels, codes)} else: - mapping = {v: c for v, c in zip(attrs["labels"], attrs["codes"])} + mapping = {v: c for v, c in zip(labels, codes)} return mapping diff --git a/tests/test_api.py b/tests/test_api.py index 2af9e61..dde9daf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,20 +20,20 @@ @pytest.mark.parametrize( "dataset_name,source,test_update", [ - # ( - # "insitu-observations-woudc-ozone-total-column-and-profiles", - # "OzoneSonde", - # False, - # ), - # ( - # "insitu-observations-woudc-ozone-total-column-and-profiles", - # "TotalOzone", - # False, - # ), - # ("insitu-observations-igra-baseline-network", "IGRA", False), - # ("insitu-observations-igra-baseline-network", "IGRA_H", False), - # ("insitu-comprehensive-upper-air-observation-network", "CUON", True), - # ("insitu-observations-gruan-reference-network", "GRUAN", False), + ( + "insitu-observations-woudc-ozone-total-column-and-profiles", + "OzoneSonde", + False, + ), + ( + "insitu-observations-woudc-ozone-total-column-and-profiles", + "TotalOzone", + False, + ), + ("insitu-observations-igra-baseline-network", "IGRA", False), + ("insitu-observations-igra-baseline-network", "IGRA_H", False), + ("insitu-comprehensive-upper-air-observation-network", "CUON", False), + ("insitu-observations-gruan-reference-network", "GRUAN", False), ( "insitu-observations-near-surface-temperature-us-climate-reference-network", "uscrn_subhourly", @@ -54,21 +54,21 @@ "uscrn_monthly", False, ), - # ( - # "insitu-observations-gnss", - # "IGS", - # False, - # ), - # ( - # "insitu-observations-gnss", - # "EPN", - # False, - # ), - # ( - # "insitu-observations-gnss", - # "IGS_R3", - # False, - # ), + ( + "insitu-observations-gnss", + "IGS", + False, + ), + ( + "insitu-observations-gnss", + "EPN", + False, + ), + ( + "insitu-observations-gnss", + "IGS_R3", + False, + ), ], ) def test_run_ingestion_pipeline( @@ -88,10 +88,16 @@ def test_run_ingestion_pipeline( update=False, ) # assert insertions have been made - counter = test_session.scalar(sa.select(sa.func.count()).select_from(Catalogue)) + counter = test_session.scalar( + sa.select(sa.func.count()) + .select_from(Catalogue) + .where(Catalogue.dataset == dataset_name) + ) assert counter > 0 # Check variables - asset = test_session.scalar(sa.select(Catalogue.asset)) + asset = test_session.scalar( + sa.select(Catalogue.asset).where(Catalogue.dataset == dataset_name) + ) s3client = S3Client.from_config(test_config.s3config) asset_filename = asset.split("/")[1] asset_local_path = Path(tmp_path, asset_filename) diff --git a/tests/test_cdm_api.py b/tests/test_cdm_api.py index c2b0181..4e316bd 100644 --- a/tests/test_cdm_api.py +++ b/tests/test_cdm_api.py @@ -115,7 +115,14 @@ def test_get_aux_fields_mapping_from_service_definition(): "soil_moisture_50cm_from_earth_surface": [], "soil_moisture_5cm_from_earth_surface": [], "soil_temperature": [ - {"auxvar": "soil_temperature_quality_flag", "metadata_name": "quality_flag"} + { + "auxvar": "soil_temperature_quality_flag", + "metadata_name": "quality_flag", + }, + { + "auxvar": "soil_temperature_processing_level", + "metadata_name": "processing_level", + }, ], "soil_temperature_100cm_from_earth_surface": [], "soil_temperature_10cm_from_earth_surface": [], @@ -147,8 +154,10 @@ def test_get_aux_fields_mapping_from_service_definition(): "minimum_solar_irradiance_quality_flag", "relative_humidity_quality_flag", "soil_temperature_quality_flag", + "soil_temperature_processing_level", "solar_irradiance_quality_flag", ] + assert not actual.var_has_uncertainty_field("accumulated_precipitation") assert actual.var_has_uncertainty_field("air_temperature") assert actual.vars_with_uncertainty_field == ["air_temperature"] diff --git a/tests/test_http_api.py b/tests/test_http_api.py index 64e0235..f846dbf 100644 --- a/tests/test_http_api.py +++ b/tests/test_http_api.py @@ -155,7 +155,12 @@ def test_get_dataset_auxiliary_variables_mapping(): "soil_moisture_20cm_from_earth_surface": [], "soil_moisture_50cm_from_earth_surface": [], "soil_moisture_5cm_from_earth_surface": [], - "soil_temperature": [], + "soil_temperature": [ + { + "auxvar": "soil_temperature_processing_level", + "metadata_name": "processing_level", + } + ], "soil_temperature_100cm_from_earth_surface": [], "soil_temperature_10cm_from_earth_surface": [], "soil_temperature_20cm_from_earth_surface": [], From 7baec7e5e559ad7d3fd2276a1c941d7d9e3ba525 Mon Sep 17 00:00:00 2001 From: garciam Date: Tue, 6 Aug 2024 09:05:27 +0200 Subject: [PATCH 20/20] fix for CI --- .github/workflows/on-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index f4b2fce..2cb9bde 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -52,7 +52,7 @@ jobs: echo TEST_INGESTION_DB_PASS=$TEST_INGESTION_DB_PASS >> .env echo CATALOGUE_PASSWORD=$CATALOGUE_PASSWORD >> .env echo STORAGE_PASSWORD=$STORAGE_PASSWORD >> .env - docker-compose up -d + docker compose up -d until pg_isready -U user -d baron -p 25432 -h localhost; do sleep 2; done - name: Download test netCDFs timeout-minutes: 2