Feature/coverage

climada/entity/exposures/litpop/nightlight.py

@@ -439,15 +439,12 @@ def read_bm_file(bm_path, filename):
         Additional info from which coordinates can be calculated.
     """
     path = Path(bm_path, filename)
-    try:
-        LOGGER.debug('Importing %s.', path)
-        curr_file = gdal.Open(str(path))
-        band1 = curr_file.GetRasterBand(1)
-        arr1 = band1.ReadAsArray()
-        del band1
-        return arr1, curr_file
-    except Exception as err:
-        raise type(err)(f"Failed to import {path} " + str(err)) from err
+    LOGGER.debug('Importing%s.', path)
+    if not path.exists():
+        raise FileNotFoundError('Invalid path: check that the path to BlackMarble file is correct.')
+    curr_file = gdal.Open(str(path))
+    arr1 = curr_file.GetRasterBand(1).ReadAsArray()
+    return arr1, curr_file
 
 def unzip_tif_to_py(file_gz):
     """Unzip image file, read it, flip the x axis, save values as pickle
@@ -469,7 +466,7 @@ def unzip_tif_to_py(file_gz):
     with gzip.open(file_gz, 'rb') as f_in:
         with file_name.open('wb') as f_out:
             shutil.copyfileobj(f_in, f_out)
-    nightlight = sparse.csc.csc_matrix(plt.imread(file_name))
+    nightlight = sparse.csc_matrix(plt.imread(file_name))
     # flip X axis
     nightlight.indices = -nightlight.indices + nightlight.shape[0] - 1
     nightlight = nightlight.tocsr()

climada/test/test_nightlight.py

@@ -20,13 +20,21 @@
 """
 
 import unittest
-import numpy as np
+import gzip
+import shutil
+import os
+from tempfile import TemporaryDirectory
 
+import numpy as np
+import scipy.sparse as sparse
 from shapely.geometry import Polygon
 
-from climada.entity.exposures.litpop import nightlight as nl_utils
+from climada.entity.exposures.litpop import nightlight
+from osgeo import gdal
+from climada.util.constants import (SYSTEM_DIR, CONFIG)
+from climada.util import files_handler
 
-from climada.util.constants import SYSTEM_DIR
+BM_FILENAMES = nightlight.BM_FILENAMES
 
 def init_test_shape():
     bounds = (14.18, 35.78, 14.58, 36.09)
@@ -49,10 +57,97 @@ def test_load_nasa_nl_shape_single_tile_pass(self):
     def test_load_nasa_nl_2016_shape_pass(self):
         """load_nasa_nl_shape_single_tile pass"""
         bounds, shape = init_test_shape()
-        nl_utils.load_nasa_nl_shape(shape, 2016, data_dir=SYSTEM_DIR, dtype=None)
-
-        # data, meta = nl_utils.load_nasa_nl_shape_single_tile(shape, path, layer=0):
-
+        nightlight.load_nasa_nl_shape(shape, 2016, data_dir=SYSTEM_DIR, dtype=None)
+
+        # data, meta = nightlight.load_nasa_nl_shape_single_tile(shape, path, layer=0):
+
+    def test_read_bm_files(self):
+        """" Test that read_bm_files function read NASA BlackMarble GeoTiff and output
+             an array and a gdal DataSet."""
+        # Create a temporary directory and the associated path
+        temp_dir = TemporaryDirectory()
+        # Download 'BlackMarble_2016_A1_geo_gray.tif' in the temporary directory and create a path
+        urls = CONFIG.exposures.litpop.nightlights.nasa_sites.list()
+        url = str(urls[0]) + 'BlackMarble_2016_A1_geo_gray.tif'
+        files_handler.download_file(url=url, download_dir=temp_dir.name)
+
+        filename = 'BlackMarble_2016_A1_geo_gray.tif'
+        # Check that the array and gdal DataSet are returned
+        with self.assertLogs('climada.entity.exposures.litpop.nightlight', level='DEBUG') as cm:
+            arr1, curr_file = nightlight.read_bm_file(bm_path=temp_dir.name, filename=filename)
+        # Check correct logging
+        self.assertIn('Importing' + temp_dir.name, cm.output[0])
+
+        # Check that the outputs are a numpy array and a gdal DataSet
+        self.assertIsInstance(arr1, np.ndarray)
+        self.assertIsInstance(curr_file, gdal.Dataset)
+        # Check that the correct band is selected
+        self.assertEqual(curr_file.GetRasterBand(1).DataType, 1)
+
+        # Release dataset, so the GC can close the file
+        curr_file = None
+
+        # Check that the right exception is raised
+        with self.assertRaises(FileNotFoundError) as cm:
+            nightlight.read_bm_file(bm_path='/Wrong/path/file.tif', filename='file.tif')
+        self.assertEqual('Invalid path: check that the path to BlackMarble file is correct.',
+                      str(cm.exception))
+
+        # delete the temporary repository
+        temp_dir.cleanup()
+
+    def test_download_nl_files(self):
+        """ Test that BlackMarble GeoTiff files are downloaded. """
+        # Create a temporary directory and the associated path
+        temp_dir = TemporaryDirectory()
+        # Test Raises
+        with self.assertRaises(ValueError) as cm:
+            nightlight.download_nl_files(req_files=np.ones(5),
+                                        files_exist=np.zeros(4),
+                                        dwnl_path=temp_dir.name)
+        self.assertEqual('The given arguments are invalid. req_files and '
+                         'files_exist must both be as long as there are files to download '
+                         '(8).', str(cm.exception))
+        with self.assertRaises(ValueError) as cm:
+            nightlight.download_nl_files(dwnl_path='not a folder')
+        self.assertEqual('The folder not a folder does not exist. Operation aborted.',
+                         str(cm.exception))
+        # Test logger
+        with self.assertLogs('climada.entity.exposures.litpop.nightlight', level='DEBUG') as cm:
+            dwl_path = nightlight.download_nl_files(req_files=np.ones(len(BM_FILENAMES),),
+                                        files_exist=np.ones(len(BM_FILENAMES),),
+                                        dwnl_path=temp_dir.name, year=2016)
+            self.assertIn('All required files already exist. No downloads necessary.', cm.output[0])
+        # Test the download
+        with self.assertLogs('climada.entity.exposures.litpop.nightlight', level='DEBUG') as cm:
+            dwl_path = nightlight.download_nl_files(req_files=np.array([1, 0, 0, 0, 0, 0, 0, 0]),
+                                        files_exist=np.array([0, 1, 1, 1, 1, 1, 1, 1]),
+                                        dwnl_path=temp_dir.name)
+            self.assertIn('Attempting to download file from '
+                        'https://eoimages.gsfc.nasa.gov/images/imagerecords/'
+                        '144000/144897/BlackMarble_2016_A1_geo_gray.tif', cm.output[0])
+            #Test if dwl_path has been returned
+            self.assertEqual(temp_dir.name, dwl_path)
+        # Delete the temporary repository
+        temp_dir.cleanup()
+
+    def test_unzip_tif_to_py(self):
+        """ Test that .gz files are unzipped and read as a sparse matrix """
+        # compress a demo .tif file to .gz
+        path_file_tif = 'data/demo/earth_engine/dresden.tif'
+        with open(path_file_tif, 'rb') as f_in:
+            with gzip.open('dresden.tif.gz','wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        # test LOGGER
+        with self.assertLogs('climada.entity.exposures.litpop.nightlight', level='INFO') as cm:
+            file_name, night=nightlight.unzip_tif_to_py('dresden.tif.gz')
+            self.assertIn('Unzipping file dresden.tif', cm.output[0])
+        # test file_name
+            self.assertEqual(str(file_name), 'dresden.tif')
+        # test the sparse matrix
+            self.assertIsInstance(night, sparse._csr.csr_matrix)
+        # delate the created demo .gz file
+        os.remove('dresden.tif.gz')
 
 # Execute Tests
 if __name__ == "__main__":