From 9b3f68c9d3e4c708f19992414ce33f990b198ea8 Mon Sep 17 00:00:00 2001 From: Adam Theisen Date: Fri, 22 Nov 2024 11:15:07 -0600 Subject: [PATCH] ADD: Adding example for bringing BNF data sources together for plotting (#881) * ADD: Adding example for bringing BNF data sources together for plotting * ENH: Updating plot a little * ENH: Updating plotting and error handling * ENH: Updating error * ENH: updating error handling * ENH: error handling * ENH: Error handling * ENH: adding additional catch * ENH: Removing error checking so we don't reveal secrets --- act/discovery/asos.py | 62 +++++----- act/io/conf/noaapsl_SurfaceMet.yaml | 83 ++++++++++++++ examples/discovery/plot_asos_temp.py | 3 +- examples/workflows/plot_amf3_comparisons.py | 118 ++++++++++++++++++++ tests/discovery/test_asos.py | 2 +- 5 files changed, 237 insertions(+), 31 deletions(-) create mode 100644 examples/workflows/plot_amf3_comparisons.py diff --git a/act/discovery/asos.py b/act/discovery/asos.py index 0d37e18d8b..fb87601c36 100644 --- a/act/discovery/asos.py +++ b/act/discovery/asos.py @@ -18,7 +18,7 @@ from urllib2 import urlopen -def get_asos_data(time_window, lat_range=None, lon_range=None, station=None): +def get_asos_data(time_window, lat_range=None, lon_range=None, station=None, regions=None): """ Returns all of the station observations from the Iowa Mesonet from either a given latitude and longitude window or a given station code. @@ -34,6 +34,9 @@ def get_asos_data(time_window, lat_range=None, lon_range=None, station=None): The longitude window to grab all of the ASOS observations from. station: str The station ID to grab the ASOS observations from. + regions: str + Region that the ASOS is in. For Alabama, it would be AL + For more than one region add it to the string with spaces between 'AL MN' Returns ------- @@ -51,33 +54,36 @@ def get_asos_data(time_window, lat_range=None, lon_range=None, station=None): """ # First query the database for all of the JSON info for every station # Only add stations whose lat/lon are within the Grid's boundaries - regions = """AF AL_ AI_ AQ_ AG_ AR_ AK AL AM_ - AO_ AS_ AR AW_ AU_ AT_ - AZ_ BA_ BE_ BB_ BG_ BO_ BR_ BF_ - BT_ BS_ BI_ BM_ BB_ BY_ BZ_ BJ_ BW_ AZ CA CA_AB - CA_BC CD_ CK_ CF_ CG_ CL_ CM_ CO CO_ CN_ CR_ CT - CU_ CV_ CY_ CZ_ DE DK_ DJ_ DM_ DO_ - DZ EE_ ET_ FK_ FM_ FJ_ FI_ FR_ GF_ PF_ - GA_ GM_ GE_ DE_ GH_ GI_ KY_ GB_ GR_ GL_ GD_ - GU_ GT_ GN_ GW_ GY_ HT_ HN_ HK_ HU_ IS_ IN_ - ID_ IR_ IQ_ IE_ IL_ IT_ CI_ JM_ JP_ - JO_ KZ_ KE_ KI_ KW_ LA_ LV_ LB_ LS_ LR_ LY_ - LT_ LU_ MK_ MG_ MW_ MY_ MV_ ML_ CA_MB - MH_ MR_ MU_ YT_ MX_ MD_ MC_ MA_ MZ_ MM_ NA_ NP_ - AN_ NL_ CA_NB NC_ CA_NF NF_ NI_ - NE_ NG_ MP_ KP_ CA_NT NO_ CA_NS CA_NU OM_ - CA_ON PK_ PA_ PG_ PY_ PE_ PH_ PN_ PL_ - PT_ CA_PE PR_ QA_ CA_QC RO_ RU_RW_ SH_ KN_ - LC_ VC_ WS_ ST_ CA_SK SA_ SN_ RS_ SC_ - SL_ SG_ SK_ SI_ SB_ SO_ ZA_ KR_ ES_ LK_ SD_ SR_ - SZ_ SE_ CH_ SY_ TW_ TJ_ TZ_ TH_ - TG_ TO_ TT_ TU TN_ TR_ TM_ UG_ UA_ AE_ UN_ UY_ - UZ_ VU_ VE_ VN_ VI_ YE_ CA_YT ZM_ ZW_ - EC_ EG_ FL GA GQ_ HI HR_ IA ID IL IO_ IN KS - KH_ KY KM_ LA MA MD ME - MI MN MO MS MT NC ND NE NH NJ NM NV NY OH OK - OR PA RI SC SV_ SD TD_ TN TX UT VA VT VG_ - WA WI WV WY""" + if regions is None: + regions = """AF AL_ AI_ AQ_ AG_ AR_ AK AL AM_ + AO_ AS_ AR AW_ AU_ AT_ + AZ_ BA_ BE_ BB_ BG_ BO_ BR_ BF_ + BT_ BS_ BI_ BM_ BB_ BY_ BZ_ BJ_ BW_ AZ CA CA_AB + CA_BC CD_ CK_ CF_ CG_ CL_ CM_ CO CO_ CN_ CR_ CT + CU_ CV_ CY_ CZ_ DE DK_ DJ_ DM_ DO_ + DZ EE_ ET_ FK_ FM_ FJ_ FI_ FR_ GF_ PF_ + GA_ GM_ GE_ DE_ GH_ GI_ KY_ GB_ GR_ GL_ GD_ + GU_ GT_ GN_ GW_ GY_ HT_ HN_ HK_ HU_ IS_ IN_ + ID_ IR_ IQ_ IE_ IL_ IT_ CI_ JM_ JP_ + JO_ KZ_ KE_ KI_ KW_ LA_ LV_ LB_ LS_ LR_ LY_ + LT_ LU_ MK_ MG_ MW_ MY_ MV_ ML_ CA_MB + MH_ MR_ MU_ YT_ MX_ MD_ MC_ MA_ MZ_ MM_ NA_ NP_ + AN_ NL_ CA_NB NC_ CA_NF NF_ NI_ + NE_ NG_ MP_ KP_ CA_NT NO_ CA_NS CA_NU OM_ + CA_ON PK_ PA_ PG_ PY_ PE_ PH_ PN_ PL_ + PT_ CA_PE PR_ QA_ CA_QC RO_ RU_RW_ SH_ KN_ + LC_ VC_ WS_ ST_ CA_SK SA_ SN_ RS_ SC_ + SL_ SG_ SK_ SI_ SB_ SO_ ZA_ KR_ ES_ LK_ SD_ SR_ + SZ_ SE_ CH_ SY_ TW_ TJ_ TZ_ TH_ + TG_ TO_ TT_ TU TN_ TR_ TM_ UG_ UA_ AE_ UN_ UY_ + UZ_ VU_ VE_ VN_ VI_ YE_ CA_YT ZM_ ZW_ + EC_ EG_ FL GA GQ_ HI HR_ IA ID IL IO_ IN KS + KH_ KY KM_ LA MA MD ME + MI MN MO MS MT NC ND NE NH NJ NM NV NY OH OK + OR PA RI SC SV_ SD TD_ TN TX UT VA VT VG_ + WA WI WV WY""" + else: + regions = '_ '.join(regions.split(' ')) networks = ['AWOS'] metadata_list = {} diff --git a/act/io/conf/noaapsl_SurfaceMet.yaml b/act/io/conf/noaapsl_SurfaceMet.yaml index 8735c1b9e7..1e48618621 100644 --- a/act/io/conf/noaapsl_SurfaceMet.yaml +++ b/act/io/conf/noaapsl_SurfaceMet.yaml @@ -543,3 +543,86 @@ mnt: long_name: Hail units: mm _type: float32 + +ctd: + info: + name: Courtland, AL + lat: + value: 34.66 + long_name: North latitude + units: degree_N + standard_name: latitude + lon: + value: 87.35 + long_name: West longitude + units: degree_W + standard_name: longitude + alt: + value: 187. + long_name: Altitude above mean sea level + units: m + standard_name: altitude + operational_date_range1: + _date_range: ['2021-09-15 18:12:00', '3000-01-01 00:00:00'] + Datalogger_ID: + _delete: True + Year: + _delete: True + J_day: + _delete: True + HoursMinutes: + _delete: True + Pressure: + long_name: Atmospheric Presure + units: mb + standard_name: air_pressure + _type: float32 + Temperature: + long_name: Atmospheric Temperature + units: degC + standard_name: air_temperature + _type: float32 + Relative_Humidity: + long_name: Atmospheric Relative_Humidity + units: percent + standard_name: relative_humidity + _type: float32 + Wind_Speed_Scalar: + long_name: Scalar Wind Speed + units: m/s + _type: float32 + Wind_Speed_Vector: + long_name: Vector Wind Speed + units: m/s + standard_name: wind_speed + _type: float32 + Wind_Direction: + long_name: Wind Direction + units: degree + standard_name: wind_from_direction + _type: float32 + Wind_Direction_STD: + long_name: Wind Direction Standard Deviation + units: degree + standard_name: wind_from_direction + cell_method: "time: standard_deviation" + _type: float32 + Battery_Voltage: + long_name: Logger Battery Voltage + units: V + _type: float32 + Solar_Radiation: + long_name: Solar Radiation + units: 'W/m^2' + Net_Radiation: + long_name: Net Radiation + units: 'W/m^2' + Precipitation: + long_name: Precipitation + units: mm + _type: float32 + Wind_Speed_Max: + long_name: Maximum Wind Speed + units: m/s + standard_name: wind_speed_of_gust + _type: float32 diff --git a/examples/discovery/plot_asos_temp.py b/examples/discovery/plot_asos_temp.py index 8e592ef54e..78d5cdd184 100644 --- a/examples/discovery/plot_asos_temp.py +++ b/examples/discovery/plot_asos_temp.py @@ -11,8 +11,7 @@ import act time_window = [datetime(2020, 2, 4, 2, 0), datetime(2020, 2, 10, 10, 0)] -station = 'KORD' -my_asoses = act.discovery.get_asos_data(time_window, station='ORD') +my_asoses = act.discovery.get_asos_data(time_window, station='ORD', regions='IL') display = act.plotting.TimeSeriesDisplay(my_asoses['ORD'], subplot_shape=(2,), figsize=(15, 10)) display.plot('temp', subplot_index=(0,)) diff --git a/examples/workflows/plot_amf3_comparisons.py b/examples/workflows/plot_amf3_comparisons.py new file mode 100644 index 0000000000..f7cd2f4a74 --- /dev/null +++ b/examples/workflows/plot_amf3_comparisons.py @@ -0,0 +1,118 @@ +""" +Consolidation of Data Sources +----------------------------- + +This example shows how to use ACT to combine multiple +datasets to support ARM's AMF3. + +""" + +import act +from datetime import datetime +import matplotlib.pyplot as plt +import numpy as np +import os + +# Get Surface Meteorology data from the ASOS stations +station = '1M4' +time_window = [datetime(2024, 10, 19), datetime(2024, 10, 24)] +ds_asos = act.discovery.get_asos_data(time_window, station=station, regions='AL')[station] +ds_asos = ds_asos.where(~np.isnan(ds_asos.tmpf), drop=True) +ds_asos['tmpf'].attrs['units'] = 'degF' +ds_asos.utils.change_units(variables='tmpf', desired_unit='degC', verbose=True) + +# Pull EPA data from AirNow +# You need an account and token from https://docs.airnowapi.org/ first +airnow_token = os.getenv('AIRNOW_API') +if airnow_token is not None and len(airnow_token) > 0: + latlon = '-87.453,34.179,-86.477,34.787' + ds_airnow = act.discovery.get_airnow_bounded_obs( + airnow_token, '2024-10-19T00', '2024-10-24T23', latlon, 'OZONE,PM25', data_type='B' + ) + ds_airnow = act.utils.convert_2d_to_1d(ds_airnow, parse='sites') + sites = ds_airnow['sites'].values + airnow = True + +# Get NOAA PSL Data from Courtland +results = act.discovery.download_noaa_psl_data( + site='ctd', instrument='Temp/RH', startdate='20241019', enddate='20241024' +) +ds_noaa = act.io.read_psl_surface_met(results) + +# Place your username and token here +username = os.getenv('ARM_USERNAME') +token = os.getenv('ARM_PASSWORD') + +# Download ARM data for the MET, OZONE, and SMPS +if username is not None and token is not None and len(username) > 1: + # Example to show how easy it is to download ARM data if a username/token are set + results = act.discovery.download_arm_data( + username, token, 'bnfmetM1.b1', '2024-10-19', '2024-10-24' + ) + ds_arm = act.io.arm.read_arm_netcdf(results) + + results = act.discovery.download_arm_data( + username, token, 'bnfaoso3M1.b1', '2024-10-19', '2024-10-24' + ) + ds_o3 = act.io.arm.read_arm_netcdf(results, cleanup_qc=True) + ds_o3.qcfilter.datafilter('o3', rm_assessments=['Suspect', 'Bad'], del_qc_var=False) + + results = act.discovery.download_arm_data( + username, token, 'bnfaossmpsM1.b1', '2024-10-19', '2024-10-24' + ) + ds_smps = act.io.arm.read_arm_netcdf(results) + + # Set up display and plot all the data + display = act.plotting.TimeSeriesDisplay( + {'ASOS': ds_asos, 'ARM': ds_arm, 'EPA': ds_airnow, 'NOAA': ds_noaa, 'ARM_O3': ds_o3}, + figsize=(12, 10), + subplot_shape=(3,), + ) + # Plot surface temperature from ASOS, NOAA, and ARM sites + title = 'Comparison of ARM MET, NOAA Courtland, and Haleyville ASOS Station' + display.plot('tmpf', dsname='ASOS', label='ASOS', subplot_index=(0,)) + display.plot('Temperature', dsname='NOAA', label='NOAA', subplot_index=(0,)) + display.plot('temp_mean', dsname='ARM', label='ARM', subplot_index=(0,), set_title=title) + display.day_night_background(dsname='ARM', subplot_index=(0,)) + + # Plot ARM and EPA Ozone data + title = 'Comparison of ARM and EPA Ozone Measurements' + display.plot('o3', dsname='ARM_O3', label='ARM', subplot_index=(1,)) + if airnow: + display.plot('OZONE_sites_1', dsname='EPA', label='EPA' + sites[1], subplot_index=(1,)) + display.plot( + 'OZONE_sites_2', + dsname='EPA', + label='EPA' + sites[2], + subplot_index=(1,), + set_title=title, + ) + display.set_yrng([0, 70], subplot_index=(1,)) + display.day_night_background(dsname='ARM', subplot_index=(1,)) + + # Plot ARM SMPS Concentrations and EPA PM2.5 data on different axes + title = 'ARM SMPS Concentrations and EPA PM2.5' + if airnow: + display.plot('PM2.5_sites_0', dsname='EPA', label='EPA ' + sites[0], subplot_index=(2,)) + display.plot( + 'PM2.5_sites_2', + dsname='EPA', + label='EPA ' + sites[2], + subplot_index=(2,), + set_title=title, + ) + display.set_yrng([0, 25], subplot_index=(2,)) + ax2 = display.axes[2].twinx() + ax2.plot(ds_smps['time'], ds_smps['total_N_conc'], label='ARM SMPS', color='purple') + ax2.set_ylabel('ARM SMPS (' + ds_smps['total_N_conc'].attrs['units'] + ')') + ax2.set_ylim([0, 7000]) + ax2.legend(loc=1) + display.day_night_background(dsname='ARM', subplot_index=(2,)) + + # Set legends + for ax in display.axes: + ax.legend(loc=2) + + plt.show() +else: + pass diff --git a/tests/discovery/test_asos.py b/tests/discovery/test_asos.py index 7bac2e8eb0..f0fba1592b 100644 --- a/tests/discovery/test_asos.py +++ b/tests/discovery/test_asos.py @@ -7,7 +7,7 @@ def test_get_ord(): time_window = [datetime(2020, 2, 4, 2, 0), datetime(2020, 2, 12, 10, 0)] - my_asoses = act.discovery.get_asos_data(time_window, station='ORD') + my_asoses = act.discovery.get_asos_data(time_window, station='ORD', regions='IL') assert 'ORD' in my_asoses.keys() assert np.all( np.equal(