for this particular time period and files, nan values were present fr…

…om the cropping. Check for those values and remove prior to interpolation, fix #197

smrf/distribute/image_data.py

@@ -1,4 +1,5 @@
 import logging
+from copy import deepcopy
 
 import numpy as np
 
@@ -223,8 +224,10 @@ def _distribute(self, data, other_attribute=None, zeros=None):
         """
 
         # get the data for the desired stations
-        # this will also order it correctly how air_temp was initialized
-        data = data[self.stations]
+        # this will also order it correctly how image_data was initialized and
+        # fill any stations not in data as NaN
+        data = deepcopy(data)
+        data = data.reindex(self.stations)
 
         if np.sum(data.isnull()) == data.shape[0]:
             raise Exception("{}: All data values are NaN"

smrf/distribute/precipitation.py

@@ -267,7 +267,7 @@ def distribute(self, data, dpt, precip_temp, ta, time, wind, temp,
         """
 
         self._logger.debug('%s Distributing all precip' % data.name)
-        data = data[self.stations]
+        data = data.reindex(self.stations)
 
         if self.config['distribution'] != 'grid':
             if self.nasde_model == 'marks2017':

smrf/spatial/grid.py

@@ -93,6 +93,8 @@ def __init__(self, config, mx, my, GridX, GridY,
         else:
             self.mask = np.ones_like(self.mx, dtype=bool)
 
+        self.metadata['mask'] = self.mask
+
     def detrendedInterpolation(self, data, flag=0, grid_method='linear'):
         """
         Interpolate using a detrended approach
@@ -102,7 +104,10 @@ def detrendedInterpolation(self, data, flag=0, grid_method='linear'):
             grid_method: scipy.interpolate.griddata interpolation method
         """
 
-        # get the trend, ensure it's positive
+        # remove nan values from the series
+        data = data.dropna()
+        if len(data) == 0:
+            raise Exception('All gridded values were NaN')
 
         if self.config['grid_local']:
             rtrend = self.detrendedInterpolationLocal(data, flag, grid_method)
@@ -123,7 +128,9 @@ def detrendedInterpolationLocal(self, data, flag=0, grid_method='linear'):
 
         # take the new full_df and fill a data column
         df = self.full_df.copy()
-        df['data'] = data[df['cell_local']].values
+        df['data'] = data.reindex(df['cell_local']).values
+        df.dropna(how='any', axis=0, inplace=True)
+
         df = df.set_index('cell_id')
         df['fit'] = df.groupby('cell_id').apply(
             lambda x: np.polyfit(x.elevation, x.data, 1))
@@ -186,7 +193,9 @@ def detrendedInterpolationMask(self, data, flag=0, grid_method='linear'):
         """
 
         # get the trend, ensure it's positive
-        pv = np.polyfit(self.mz[self.mask].astype(float), data[self.mask], 1)
+        data_mask = data[self.metadata['mask']]
+        elevation_mask = self.metadata.loc[data_mask.index, 'elevation']
+        pv = np.polyfit(elevation_mask, data_mask, 1)
 
         # apply trend constraints
         if flag == 1 and pv[0] < 0:
@@ -197,14 +206,16 @@ def detrendedInterpolationMask(self, data, flag=0, grid_method='linear'):
         self.pv = pv
 
         # detrend the data
-        el_trend = self.mz * pv[0] + pv[1]
-        dtrend = data - el_trend
+        el_trend = elevation_mask * pv[0] + pv[1]
+        dtrend = data_mask - el_trend
 
         # interpolate over the DEM grid
-        idtrend = griddata((self.mx, self.my),
-                           dtrend,
-                           (self.GridX, self.GridY),
-                           method=grid_method)
+        idtrend = griddata(
+            (self.metadata.loc[data_mask.index, 'utm_x'],
+             self.metadata.loc[data_mask.index, 'utm_y']),
+            dtrend,
+            (self.GridX, self.GridY),
+            method=grid_method)
 
         # retrend the data
         rtrend = idtrend + pv[0]*self.GridZ + pv[1]

smrf/tests/basins/Lakes/config_issue_197.ini

@@ -90,6 +90,7 @@ wind_ninja_tz:                  UTC
 [precip]
 distribution:                   grid
 grid_method:                    cubic
+#grid_local:                     True
 detrend:                        True
 detrend_slope:                  1
 grid_mask:                      False
@@ -182,4 +183,5 @@ out_location:                  ./output
 threading:                     False
 queue_max_values:              1
 time_out:                      None
-log_level:                     debug
+log_level:                     debug
+log_file:                      ./output/log.txt

smrf/tests/data/test_hrrr.py

@@ -102,32 +102,7 @@ def test_load_timestep_threaded(self):
 
 class TestLoadHRRRLakes(SMRFTestCaseLakes):
 
-    # @classmethod
-    # def setUpClass(cls):
-    #     super().setUpClass()
-
-    #     config = cls.base_config_copy()
-
-    #     adj_config = {
-    #         'time': {
-    #             'start_date': '2018-12-04 13:00',
-    #             'end_date': '2018-12-04 13:30',
-    #             'time_zone': 'utc'
-    #         },
-    #         'system': {
-    #             'threading': False,
-    #             'log_file': None
-    #         },
-    #     }
-    #     config.raw_cfg.update(adj_config)
-    #     config.apply_recipes()
-    #     config = cast_all_variables(config, config.mcfg)
-
-    #     cls.config = config
-
     def test_197_image_data_index(self):
 
-        # run_smrf(self.config)
         config_file = os.path.join(self.basin_dir, 'config_issue_197.ini')
         run_smrf(config_file)
-        self.config