Make notebook testing multithreaded

[jrleeman]

Going to see if this will speed up our testing runs.

[lesserwhirls]

Is there a risk of this failing due to netCDF4 not being threads safe?

[jrleeman]

I don't think so - since it's really multiple processes instead of threads (I decided to use multiprocessing instead of threading)

[lesserwhirls]

Ah 👍

[jrleeman]

Looks like the Mac builds are backing up - will check in later and see what's going on with Travis

[jrleeman]

FYI @mjames-upc we've had to blacklist a couple of AWIPS notebooks (NEXRAD3_Storm_Total_Accumulation.ipynb and Watch_and_Warning_Polygons.ipynb) that have been breaking things. Wanted to make sure you knew and could have a look at them.

[mjames-upc]

@jrleeman what should I be looking at for this? travis build logs aren't showing me anything.

[dopplershift]

@mjames-upc

[NbConvertApp] ERROR | Error while converting 'notebooks/python-awips/Watch_and_Warning_Polygons.ipynb'
Traceback (most recent call last):
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/nbconvertapp.py", line 373, in export_single_notebook
    output, resources = self.exporter.from_filename(notebook_filename, resources=resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 171, in from_filename
    return self.from_file(f, resources=resources, **kw)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 189, in from_file
    return self.from_notebook_node(nbformat.read(file_stream, as_version=4), resources=resources, **kw)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/notebook.py", line 31, in from_notebook_node
    nb_copy, resources = super(NotebookExporter, self).from_notebook_node(nb, resources, **kw)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 131, in from_notebook_node
    nb_copy, resources = self._preprocess(nb_copy, resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 308, in _preprocess
    nbc, resc = preprocessor(nbc, resc)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/base.py", line 47, in __call__
    return self.preprocess(nb,resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/execute.py", line 211, in preprocess
    nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/base.py", line 70, in preprocess
    nb.cells[index], resources = self.preprocess_cell(cell, resources, index)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/execute.py", line 242, in preprocess_cell
    raise CellExecutionError(msg)
nbconvert.preprocessors.execute.CellExecutionError: An error occurred while executing the following cell:
------------------
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
import cartopy.feature as cfeat
from cartopy.feature import ShapelyFeature,NaturalEarthFeature
from shapely.geometry import Polygon
from awips.tables import vtec
def warning_color(phensig):
    if vtec[phensig]['color']:
        return vtec[phensig]['color']
def make_map(bbox, projection=ccrs.PlateCarree()):
    fig, ax = plt.subplots(figsize=(20,12),
            subplot_kw=dict(projection=projection))
    ax.set_extent(bbox)
    ax.coastlines(resolution='50m')
    gl = ax.gridlines(draw_labels=True)
    gl.xlabels_top = gl.ylabels_right = False
    gl.xformatter = LONGITUDE_FORMATTER
    gl.yformatter = LATITUDE_FORMATTER
    return fig, ax
%matplotlib inline
bbox=[-127,-64,24,49]
fig, ax = make_map(bbox=bbox)
for i, geom in enumerate(geometries):
    bounds = Polygon(geom)
    intersection = bounds.intersection
    geoms = (intersection(geom)
         for geom in geometries
         if bounds.intersects(geom))
    
    # For now do not plot SPS
    if (parameters['phensig'][i]):
        vparm = parameters['phensig'][i]
        color = warning_color(vparm.astype('U13'))
        shape_feature = ShapelyFeature(geoms,ccrs.PlateCarree(), 
                        facecolor='none', edgecolor=color)
        ax.add_feature(shape_feature)
political_boundaries = NaturalEarthFeature(category='cultural',
                               name='admin_0_boundary_lines_land',
                               scale='50m', facecolor='none')
states = NaturalEarthFeature(category='cultural',
                               name='admin_1_states_provinces_lakes',
                               scale='50m', facecolor='none')
ax.add_feature(political_boundaries, linestyle='-')
ax.add_feature(states, linestyle=':')
plt.show()
------------------
TopologicalError: The operation 'GEOSIntersection_r' could not be performed. Likely cause is invalidity of the geometry <shapely.geometry.polygon.Polygon object at 0x7f114efeb320>

and

[NbConvertApp] ERROR | Error while converting 'notebooks/python-awips/NEXRAD3_Storm_Total_Accumulation.ipynb'
Traceback (most recent call last):
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/nbconvertapp.py", line 373, in export_single_notebook
    output, resources = self.exporter.from_filename(notebook_filename, resources=resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 171, in from_filename
    return self.from_file(f, resources=resources, **kw)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 189, in from_file
    return self.from_notebook_node(nbformat.read(file_stream, as_version=4), resources=resources, **kw)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/notebook.py", line 31, in from_notebook_node
    nb_copy, resources = super(NotebookExporter, self).from_notebook_node(nb, resources, **kw)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 131, in from_notebook_node
    nb_copy, resources = self._preprocess(nb_copy, resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/exporters/exporter.py", line 308, in _preprocess
    nbc, resc = preprocessor(nbc, resc)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/base.py", line 47, in __call__
    return self.preprocess(nb,resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/execute.py", line 211, in preprocess
    nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/base.py", line 70, in preprocess
    nb.cells[index], resources = self.preprocess_cell(cell, resources, index)
  File "/home/travis/miniconda/envs/unidata-workshop/lib/python3.6/site-packages/nbconvert/preprocessors/execute.py", line 242, in preprocess_cell
    raise CellExecutionError(msg)
nbconvert.preprocessors.execute.CellExecutionError: An error occurred while executing the following cell:
------------------
if response.getData():
    for record in response.getData():
        # Get hdf5
        idra = record.getHdf5Data()
        rdat,azdat,depVals,threshVals = RadarCommon.get_hdf5_data(idra)
        dim = rdat.getDimension()
        lat,lon = float(record.getLatitude()),float(record.getLongitude())
        radials,rangeGates = rdat.getSizes()
        
        # Convert raw byte to pixel value
        rawValue=np.array(rdat.getByteData())
        array = []
        for rec in rawValue:
            if rec<0:
                rec+=256
            array.append(rec)
        #array=rawValue
            
        if azdat:
            azVals = azdat.getFloatData()
            az = np.array(RadarCommon.encode_radial(azVals))
            dattyp = RadarCommon.get_data_type(azdat)
            az = np.append(az,az[-1])
        header = RadarCommon.get_header(record, format, rangeGates, radials, azdat, 'description')
        rng = np.linspace(0, rangeGates, rangeGates + 1)
        # Convert az/range to a lat/lon
        from pyproj import Geod
        g = Geod(ellps='clrk66')
        center_lat = np.ones([len(az),len(rng)])*lat
        center_lon = np.ones([len(az),len(rng)])*lon
        az2D = np.ones_like(center_lat)*az[:,None]
        rng2D = np.ones_like(center_lat)*np.transpose(rng[:,None])*nexrad[code]['res']
        lons,lats,back=g.fwd(center_lon,center_lat,az2D,rng2D)
        bbox = [lons.min(), lons.max(), lats.min(), lats.max()]
        # Create 2d array
        multiArray = np.reshape(array, (-1, rangeGates))
        data = ma.array(multiArray)
        if product==135:
            data[data<2] = ma.masked
        elif product==172:
            # <entry displayValue='0.0' pixelValue='6' />
            # <entry displayValue='0.1' pixelValue='16' />
            # <entry displayValue='0.3' pixelValue='32' />
            # <entry displayValue='0.6' pixelValue='48' />
            # <entry displayValue='1.0' pixelValue='64' />
            # <entry displayValue='1.5' pixelValue='80' />
            # <entry displayValue='2.0' pixelValue='96' />
            # <entry displayValue='2.5' pixelValue='112' />
            # <entry displayValue='3.0' pixelValue='128' />
            # <entry displayValue='4.0' pixelValue='144' />
            # <entry displayValue='5.0' pixelValue='160' />
            # <entry displayValue='6.0' pixelValue='176' />
            # <entry displayValue='8.0' pixelValue='192' />
            # <entry displayValue='10.0' pixelValue='208' />
            # <entry displayValue='12.0' pixelValue='224' />
            # <entry displayValue='15.0' pixelValue='240' />
            # <entry displayValue='31.0' pixelValue='255' />
            
            # The Scale and Offset used in the equation (F = (N - OFFSET) / SCALE), where 
            # N is the integer data value and F is the resulting floating point value) 
            # to relate the integer data values to physical units are ANSI/IEEE
            # Standard 754-1985 floating point values. Halfwords 31 and 32 contain the 
            # Scale, and halfwords 33 and 34 contain the Offset. 
            data = ma.array((multiArray - float(threshVals[1])/float(threshVals[0])))
            data[data==0] = ma.masked
        else:
            data = ma.array(threshVals[0]/10. + (data)*threshVals[1]/10.)
            if nexrad[code]['unit'] == 'kts':
                data[data<-63] = ma.masked
                data *= 1.94384 # Convert to knots
            else:
                data[data<=((threshVals[0]/10.)+threshVals[1]/10.)] = ma.masked
        
        # Save our requested grids so we can render them multiple times
        radar_grid = {
            "code": code,
            "bbox": bbox,
            "lats": lats,
            "lons": lons,
            "data": data
        }
        grids.append(radar_grid)
------------------
AttributeError: 'list' object has no attribute 'getDimension'