Merge pull request #190 from jrleeman/multiprocessing

Make notebook testing multithreaded

.travis.yml

@@ -35,7 +35,6 @@ install:
   - source activate unidata-workshop
   - python --version
   - conda list
-  - python -m ipykernel install --user --name workshop
 
 script:
   - python run_notebooks.py

run_notebooks.py

@@ -6,29 +6,41 @@
 NOTEBOOKS_DIR = 'notebooks'
 SKIP_NOTEBOOKS = [os.path.join('Bonus','What to do when things go wrong.ipynb'),
                   os.path.join('python-awips',
-                                'NEXRAD3_Storm_Total_Accumulation.ipynb')]
+                                'NEXRAD3_Storm_Total_Accumulation.ipynb'),
+                  os.path.join('python-awips',
+                                'Watch_and_Warning_Polygons.ipynb')]
 
 
 def run_notebook(notebook):
     args = ['jupyter', 'nbconvert', '--execute',
             '--ExecutePreprocessor.timeout=900',
-            '--ExecutePreprocessor.kernel_name=workshop',
+            '--ExecutePreprocessor.kernel_name=python3',
             '--to=notebook', '--stdout']
 
     args.append(notebook)
     with subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=None) as proc:
         proc.wait()
         return proc.returncode
 
+results = []
+def log_result(result):
+    results.append(result)
 
 if __name__ == '__main__':
     import glob
+    import multiprocessing as mp
     import sys
 
     ret = 0
     notebooks = set(glob.glob(os.path.join(NOTEBOOKS_DIR, '**', '*.ipynb'), recursive=True))
     notebooks -= set(os.path.join(NOTEBOOKS_DIR, s)
                      for s in SKIP_NOTEBOOKS)
-    for path in sorted(notebooks):
-        ret = max(run_notebook(path), ret)
+
+    with mp.Pool(processes=6) as pool:
+        for notebook in notebooks:
+            pool.apply_async(run_notebook, args=(notebook,), callback=log_result)
+        pool.close()
+        pool.join()
+
+    ret = max(results)
     sys.exit(ret)