TST: Reorganize the entire test module. (ARM-DOE#771)

* TST: Reorganizing corrections tests.

* TST: Rearrange discovery tests into submodules.

* FIX: Fix missing import missed in init as well as update tests and
linting.

* TST: Reorganizing the io test module.

* TST: Reorganize plotting module.

* TST: Reorganizing qc test module.

* TST: Reorganize the retrievals tests.

* TST: Reorganize the utils test module.

* STY: Fix for PEP8 lines.

* FIX: Update names so no overlap.

* FIX: Move baseline.

* DEL: Remove missed file.

* FIX: fix for index issue in test_plot.

* FIX: Attempted fix for missing codecov.

act/discovery/asos.py

@@ -50,9 +50,6 @@ def get_asos_data(time_window, lat_range=None, lon_range=None, station=None):
         $ station = "KORD"
         $ my_asoses = act.discovery.get_asos(time_window, station="ORD")
     """
-    message = 'act.discovery.get_asos will be renamed in version 2.0.0 to act.discovery.asos'
-    warnings.warn(message, DeprecationWarning, 2)
-
     # First query the database for all of the JSON info for every station
     # Only add stations whose lat/lon are within the Grid's boundaries
     regions = """AF AL_ AI_ AQ_ AG_ AR_ AK AL AM_

act/io/__init__.py

@@ -36,6 +36,7 @@
             'read_psl_wind_profiler_temperature',
             'read_psl_parsivel',
             'read_psl_radar_fmcw_moment',
+            'read_psl_surface_met',
         ],
         'pysp2': ['read_hk_file', 'read_sp2', 'read_sp2_dat'
         ],

act/retrievals/__init__.py

@@ -23,6 +23,7 @@
             'calculate_pbl_liu_liang',
             'calculate_precipitable_water',
             'calculate_stability_indicies',
+            'calculate_pbl_heffter',
         ],
         'sp2': ['calc_sp2_diams_masses', 'process_sp2_psds'],
     },

act/tests/corrections/test_ceil.py

@@ -0,0 +1,20 @@
+import numpy as np
+import xarray as xr
+
+import act
+
+
+def test_correct_ceil():
+    # Make a fake ARM dataset to test with, just an array with 1e-7 for half
+    # of it
+    fake_data = 10 * np.ones((300, 20))
+    fake_data[:, 10:] = -1
+    arm_ds = {}
+    arm_ds['backscatter'] = xr.DataArray(fake_data)
+    arm_ds = act.corrections.ceil.correct_ceil(arm_ds)
+    assert np.all(arm_ds['backscatter'].data[:, 10:] == -7)
+    assert np.all(arm_ds['backscatter'].data[:, 1:10] == 1)
+
+    arm_ds['backscatter'].attrs['units'] = 'dummy'
+    arm_ds = act.corrections.ceil.correct_ceil(arm_ds)
+    assert arm_ds['backscatter'].units == 'log(dummy)'

act/tests/corrections/test_doppler_lidar.py

@@ -0,0 +1,19 @@
+import numpy as np
+
+import act
+
+
+def test_correct_dl():
+    # Test the DL correction script on a PPI dataset eventhough it will
+    # mostlikely be used on FPT scans. Doing this to save space with only
+    # one datafile in the repo.
+    files = act.tests.sample_files.EXAMPLE_DLPPI
+    ds = act.io.arm.read_arm_netcdf(files)
+
+    new_ds = act.corrections.doppler_lidar.correct_dl(ds, fill_value=np.nan)
+    data = new_ds['attenuated_backscatter'].values
+    np.testing.assert_almost_equal(np.nansum(data), -186479.83, decimal=0.1)
+
+    new_ds = act.corrections.doppler_lidar.correct_dl(ds, range_normalize=False)
+    data = new_ds['attenuated_backscatter'].values
+    np.testing.assert_almost_equal(np.nansum(data), -200886.0, decimal=0.1)

act/tests/corrections/test_mpl_corrections.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+import act
+
+
+def test_correct_mpl():
+    # Make a fake ARM dataset to test with, just an array with 1e-7 for half
+    # of it
+    test_data = act.io.arm.read_arm_netcdf(act.tests.EXAMPLE_MPL_1SAMPLE)
+    ds = act.corrections.mpl.correct_mpl(test_data)
+    sig_cross_pol = ds['signal_return_cross_pol'].values[1, 10:15]
+    sig_co_pol = ds['signal_return_co_pol'].values[1, 10:15]
+    height = ds['height'].values[0:10]
+    overlap0 = ds['overlap_correction'].values[1, 0, 0:5]
+    overlap1 = ds['overlap_correction'].values[1, 1, 0:5]
+    overlap2 = ds['overlap_correction'].values[1, 2, 0:5]
+    np.testing.assert_allclose(overlap0, [0.0, 0.0, 0.0, 0.0, 0.0])
+    np.testing.assert_allclose(overlap1, [754.338, 754.338, 754.338, 754.338, 754.338])
+    np.testing.assert_allclose(overlap2, [181.9355, 181.9355, 181.9355, 181.9355, 181.9355])
+    np.testing.assert_allclose(
+        sig_cross_pol,
+        [-0.5823283, -1.6066532, -1.7153032, -2.520143, -2.275405],
+        rtol=4e-06,
+    )
+    np.testing.assert_allclose(
+        sig_co_pol, [12.5631485, 11.035495, 11.999875, 11.09393, 11.388968], rtol=1e-6
+    )
+    np.testing.assert_allclose(
+        height,
+        [
+            0.00749012,
+            0.02247084,
+            0.03745109,
+            0.05243181,
+            0.06741206,
+            0.08239277,
+            0.09737302,
+            0.11235374,
+            0.12733398,
+            0.14231472,
+        ],
+        rtol=1e-6,
+    )
+    assert ds['signal_return_co_pol'].attrs['units'] == '10 * log10(count/us)'
+    assert ds['signal_return_cross_pol'].attrs['units'] == '10 * log10(count/us)'
+    assert ds['cross_co_ratio'].attrs['long_name'] == 'Cross-pol / Co-pol ratio * 100'
+    assert ds['cross_co_ratio'].attrs['units'] == '1'
+    assert 'description' not in ds['cross_co_ratio'].attrs.keys()
+    assert 'ancillary_variables' not in ds['cross_co_ratio'].attrs.keys()
+    assert np.all(np.round(ds['cross_co_ratio'].data[0, 500]) == 34.0)
+    assert np.all(np.round(ds['signal_return_co_pol'].data[0, 11]) == 11)
+    assert np.all(np.round(ds['signal_return_co_pol'].data[0, 500]) == -6)
+    test_data.close()
+    ds.close()

act/tests/corrections/test_raman_lidar.py

@@ -0,0 +1,20 @@
+import numpy as np
+
+import act
+
+
+def test_correct_rl():
+    # Using ceil data in RL place to save memory
+    files = act.tests.sample_files.EXAMPLE_RL1
+    ds = act.io.arm.read_arm_netcdf(files)
+
+    ds = act.corrections.raman_lidar.correct_rl(ds, range_normalize_log_values=True)
+    np.testing.assert_almost_equal(
+        np.max(ds['depolarization_counts_high'].values), 9.91, decimal=2
+    )
+    np.testing.assert_almost_equal(
+        np.min(ds['depolarization_counts_high'].values), -7.00, decimal=2
+    )
+    np.testing.assert_almost_equal(
+        np.mean(ds['depolarization_counts_high'].values), -1.45, decimal=2
+    )

act/tests/corrections/test_ship.py

@@ -0,0 +1,16 @@
+import xarray as xr
+
+import act
+
+
+def test_correct_wind():
+    nav = act.io.arm.read_arm_netcdf(act.tests.sample_files.EXAMPLE_NAV)
+    nav = act.utils.ship_utils.calc_cog_sog(nav)
+
+    aosmet = act.io.arm.read_arm_netcdf(act.tests.sample_files.EXAMPLE_AOSMET)
+
+    ds = xr.merge([nav, aosmet], compat='override')
+    ds = act.corrections.ship.correct_wind(ds)
+
+    assert round(ds['wind_speed_corrected'].values[800]) == 5.0
+    assert round(ds['wind_direction_corrected'].values[800]) == 92.0

act/tests/discovery/test_airnow.py

@@ -0,0 +1,53 @@
+import os
+
+import numpy as np
+
+import act
+
+
+def test_get_airnow():
+    token = os.getenv('AIRNOW_API')
+    if token is not None:
+        if len(token) == 0:
+            return
+        results = act.discovery.get_airnow_forecast(token, '2022-05-01', zipcode=60108, distance=50)
+        assert results['CategoryName'].values[0] == 'Good'
+        assert results['AQI'].values[2] == -1
+        assert results['ReportingArea'].values[3] == 'Aurora and Elgin'
+
+        results = act.discovery.get_airnow_forecast(
+            token, '2022-05-01', distance=50, latlon=[41.958, -88.12]
+        )
+        assert results['CategoryName'].values[3] == 'Good'
+        assert results['AQI'].values[2] == -1
+        assert results['ReportingArea'][3] == 'Aurora and Elgin'
+
+        results = act.discovery.get_airnow_obs(token, date='2022-05-01', zipcode=60108, distance=50)
+        assert results['AQI'].values[0] == 26
+        assert results['ParameterName'].values[1] == 'PM2.5'
+        assert results['CategoryName'].values[0] == 'Good'
+
+        results = act.discovery.get_airnow_obs(token, zipcode=60108, distance=50)
+        assert results['ReportingArea'].values[0] == 'Aurora and Elgin'
+        results = act.discovery.get_airnow_obs(token, latlon=[41.958, -88.12], distance=50)
+        assert results['StateCode'].values[0] == 'IL'
+
+        with np.testing.assert_raises(NameError):
+            results = act.discovery.get_airnow_obs(token)
+        with np.testing.assert_raises(NameError):
+            results = act.discovery.get_airnow_forecast(token, '2022-05-01')
+
+        results = act.discovery.get_airnow_obs(
+            token, date='2022-05-01', distance=50, latlon=[41.958, -88.12]
+        )
+        assert results['AQI'].values[0] == 26
+        assert results['ParameterName'].values[1] == 'PM2.5'
+        assert results['CategoryName'].values[0] == 'Good'
+
+        lat_lon = '-88.245401,41.871346,-87.685099,42.234359'
+        results = act.discovery.get_airnow_bounded_obs(
+            token, '2022-05-01T00', '2022-05-01T12', lat_lon, 'OZONE,PM25', data_type='B'
+        )
+        assert results['PM2.5'].values[-1, 0] == 1.8
+        assert results['OZONE'].values[0, 0] == 37.0
+        assert len(results['time'].values) == 13

act/tests/discovery/test_arm_discovery.py

@@ -0,0 +1,115 @@
+import glob
+import os
+
+import numpy as np
+
+import act
+
+
+def test_download_armdata():
+    if not os.path.isdir(os.getcwd() + '/data/'):
+        os.makedirs(os.getcwd() + '/data/')
+
+    # Place your username and token here
+    username = os.getenv('ARM_USERNAME')
+    token = os.getenv('ARM_PASSWORD')
+
+    if username is not None and token is not None:
+        if len(username) == 0 and len(token) == 0:
+            return
+        datastream = 'sgpmetE13.b1'
+        startdate = '2020-01-01'
+        enddate = startdate
+        outdir = os.getcwd() + '/data/'
+
+        results = act.discovery.arm.download_arm_data(
+            username, token, datastream, startdate, enddate, output=outdir
+        )
+        files = glob.glob(outdir + datastream + '*20200101*cdf')
+        if len(results) > 0:
+            assert files is not None
+            assert 'sgpmetE13' in files[0]
+
+        if files is not None:
+            if len(files) > 0:
+                os.remove(files[0])
+
+        datastream = 'sgpmeetE13.b1'
+        act.discovery.arm.download_arm_data(
+            username, token, datastream, startdate, enddate, output=outdir
+        )
+        files = glob.glob(outdir + datastream + '*20200101*cdf')
+        assert len(files) == 0
+
+        with np.testing.assert_raises(ConnectionRefusedError):
+            act.discovery.arm.download_arm_data(
+                username, token + '1234', datastream, startdate, enddate, output=outdir
+            )
+
+        datastream = 'sgpmetE13.b1'
+        results = act.discovery.arm.download_arm_data(
+            username, token, datastream, startdate, enddate
+        )
+        assert len(results) == 1
+
+
+def test_download_armdata_hourly():
+    if not os.path.isdir(os.getcwd() + '/data/'):
+        os.makedirs(os.getcwd() + '/data/')
+
+    # Place your username and token here
+    username = os.getenv('ARM_USERNAME')
+    token = os.getenv('ARM_PASSWORD')
+
+    if username is not None and token is not None:
+        if len(username) == 0 and len(token) == 0:
+            return
+        datastream = 'sgpmetE13.b1'
+        startdate = '2020-01-01T00:00:00'
+        enddate = '2020-01-01T12:00:00'
+        outdir = os.getcwd() + '/data/'
+
+        results = act.discovery.arm.download_arm_data(
+            username, token, datastream, startdate, enddate, output=outdir
+        )
+        files = glob.glob(outdir + datastream + '*20200101*cdf')
+        if len(results) > 0:
+            assert files is not None
+            assert 'sgpmetE13' in files[0]
+
+        if files is not None:
+            if len(files) > 0:
+                os.remove(files[0])
+
+        datastream = 'sgpmeetE13.b1'
+        act.discovery.arm.download_arm_data(
+            username, token, datastream, startdate, enddate, output=outdir
+        )
+        files = glob.glob(outdir + datastream + '*20200101*cdf')
+        assert len(files) == 0
+
+        with np.testing.assert_raises(ConnectionRefusedError):
+            act.discovery.arm.download_arm_data(
+                username, token + '1234', datastream, startdate, enddate, output=outdir
+            )
+
+        datastream = 'sgpmetE13.b1'
+        results = act.discovery.arm.download_arm_data(
+            username, token, datastream, startdate, enddate
+        )
+        assert len(results) == 1
+
+
+def test_arm_doi():
+    datastream = 'sgpmetE13.b1'
+    startdate = '2022-01-01'
+    enddate = '2022-12-31'
+    doi = act.discovery.get_arm_doi(datastream, startdate, enddate)
+
+    assert len(doi) > 10
+    assert isinstance(doi, str)
+    assert 'doi' in doi
+    assert 'Kyrouac' in doi
+
+    doi = act.discovery.get_arm_doi('test', startdate, enddate)
+    assert "No DOI Found" in doi

act/tests/discovery/test_asos.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+
+import numpy as np
+
+import act
+
+
+def test_get_ord():
+    time_window = [datetime(2020, 2, 4, 2, 0), datetime(2020, 2, 12, 10, 0)]
+    my_asoses = act.discovery.get_asos_data(time_window, station='ORD')
+    assert 'ORD' in my_asoses.keys()
+    assert np.all(
+        np.equal(
+            my_asoses['ORD']['sknt'].values[:10],
+            np.array([13.0, 11.0, 14.0, 14.0, 13.0, 11.0, 14.0, 13.0, 13.0, 13.0]),
+        )
+    )
+
+
+def test_get_region():
+    my_keys = ['MDW', 'IGQ', 'ORD', '06C', 'PWK', 'LOT', 'GYY']
+    time_window = [datetime(2020, 2, 4, 2, 0), datetime(2020, 2, 12, 10, 0)]
+    lat_window = (41.8781 - 0.5, 41.8781 + 0.5)
+    lon_window = (-87.6298 - 0.5, -87.6298 + 0.5)
+    my_asoses = act.discovery.get_asos_data(time_window, lat_range=lat_window, lon_range=lon_window)
+    asos_keys = list(my_asoses.keys())
+    assert asos_keys == my_keys

act/tests/discovery/test_cropscape.py

@@ -0,0 +1,20 @@
+import act
+
+
+def test_croptype():
+    year = 2018
+    lat = 37.15
+    lon = -98.362
+    # Try for when the cropscape API is not working
+    try:
+        crop = act.discovery.cropscape.get_crop_type(lat, lon, year)
+        crop2 = act.discovery.cropscape.get_crop_type(lat, lon)
+    except Exception:
+        return
+
+    # print(crop, crop2)
+    if crop is not None:
+        assert crop == 'Dbl Crop WinWht/Sorghum'
+    if crop2 is not None:
+        # assert crop2 == 'Sorghum'
+        assert crop2 == 'Soybeans'