From ee0b31fa48827923eecb5d6ec546c63081d999e2 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Tue, 3 Sep 2024 14:34:09 +0000 Subject: [PATCH 01/15] feature : Add the flash_age compositor into the /etc/composites/li.yaml files --- satpy/etc/composites/li.yaml | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/satpy/etc/composites/li.yaml b/satpy/etc/composites/li.yaml index 4d3cc88e95..19e879590c 100644 --- a/satpy/etc/composites/li.yaml +++ b/satpy/etc/composites/li.yaml @@ -10,69 +10,78 @@ composites: compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: acc_flash prerequisites: - - flash_accumulation + - flash_accumulation acc_flash_alpha: description: Composite to colorise the AF product using the flash accumulation with transparency compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: acc_flash_alpha prerequisites: - - flash_accumulation + - flash_accumulation acc_flash_area: description: Composite to colorise the AFA product using the flash area compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: acc_flash_area prerequisites: - - accumulated_flash_area + - accumulated_flash_area acc_flash_area_alpha: description: Composite to colorise the AFA product using the flash area with transparency compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: acc_flash_area_alpha prerequisites: - - accumulated_flash_area + - accumulated_flash_area acc_flash_radiance: description: Composite to colorise the AFR product using the flash radiance compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: lightning_radiance prerequisites: - - flash_radiance + - flash_radiance acc_flash_radiance_alpha: description: Composite to colorise the AFR product using the flash radiance with transparency compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: lightning_radiance_alpha prerequisites: - - flash_radiance + - flash_radiance flash_radiance: description: Composite to colorise the LFL product using the flash radiance compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: lightning_radiance prerequisites: - - radiance + - radiance flash_radiance_alpha: description: Composite to colorise the LFL product using the flash radiance with transparency compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: lightning_radiance_alpha prerequisites: - - radiance + - radiance group_radiance: description: Composite to colorise the LGR product using the flash radiance compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: lightning_radiance prerequisites: - - radiance + - radiance group_radiance_alpha: description: Composite to colorise the LGR product using the flash radiance with transparency compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: lightning_radiance_alpha prerequisites: - - radiance + - radiance # DEPRECATED, USE acc_flash_area INSTEAD flash_area: compositor: !!python/name:satpy.composites.SingleBandCompositor standard_name: acc_flash_area prerequisites: - - accumulated_flash_area + - accumulated_flash_area + + flash_age: + description: Composite to colorise the LFL product using the flash time + compositor: !!python/name:satpy.composites.lightning.LightningTimeCompositor + standard_name: lightning_time + time_range: 60 # range for colormap in minutes + reference_time: end_time + prerequisites: + - flash_time From 2ffffc31d0ab0645ab4f5d40a4e7305f51d08126 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Wed, 4 Sep 2024 08:45:32 +0000 Subject: [PATCH 02/15] feature : Add the flash_age composite for li datas --- satpy/etc/enhancements/li.yaml | 94 +++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/satpy/etc/enhancements/li.yaml b/satpy/etc/enhancements/li.yaml index 49009808eb..9aaa5c4a0b 100644 --- a/satpy/etc/enhancements/li.yaml +++ b/satpy/etc/enhancements/li.yaml @@ -1,60 +1,84 @@ enhancements: -# note that the colormap parameters are tuned for 5 minutes of files accumulation -# these are tentative recipes that will need to be further tuned as we gain experience with LI data + # note that the colormap parameters are tuned for 5 minutes of files accumulation + # these are tentative recipes that will need to be further tuned as we gain experience with LI data acc_flash: standard_name: acc_flash operations: - - name: colorize - method: !!python/name:satpy.enhancements.colorize - kwargs: - palettes: - - {colors: ylorrd, min_value: 0, max_value: 5} + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - { colors: ylorrd, min_value: 0, max_value: 5 } acc_flash_alpha: standard_name: acc_flash_alpha operations: - - name: colorize - method: !!python/name:satpy.enhancements.colorize - kwargs: - palettes: - - {colors: ylorrd, min_value: 0, max_value: 5, - min_alpha: 120, max_alpha: 180} + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - { + colors: ylorrd, + min_value: 0, + max_value: 5, + min_alpha: 120, + max_alpha: 180, + } acc_flash_area: standard_name: acc_flash_area operations: - - name: colorize - method: !!python/name:satpy.enhancements.colorize - kwargs: - palettes: - - {colors: ylorrd, min_value: 0, max_value: 20} + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - { colors: ylorrd, min_value: 0, max_value: 20 } acc_flash_area_alpha: standard_name: acc_flash_area_alpha operations: - - name: colorize - method: !!python/name:satpy.enhancements.colorize - kwargs: - palettes: - - {colors: ylorrd, min_value: 0, max_value: 20, - min_alpha: 120, max_alpha: 180} + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - { + colors: ylorrd, + min_value: 0, + max_value: 20, + min_alpha: 120, + max_alpha: 180, + } lightning_radiance: standard_name: lightning_radiance operations: - - name: colorize - method: !!python/name:satpy.enhancements.colorize - kwargs: - palettes: - - {colors: ylorrd, min_value: 0, max_value: 1000} + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - { colors: ylorrd, min_value: 0, max_value: 1000 } lightning_radiance_alpha: standard_name: lightning_radiance_alpha operations: - - name: colorize - method: !!python/name:satpy.enhancements.colorize - kwargs: - palettes: - - {colors: ylorrd, min_value: 0, max_value: 1000, - min_alpha: 120, max_alpha: 180} + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - { + colors: ylorrd, + min_value: 0, + max_value: 1000, + min_alpha: 120, + max_alpha: 180, + } + + lightning_time: + standard_name: lightning_time + operations: + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - { colors: ylorrd, min_value: 0, max_value: 1 } From 1e7db424bbea71b865575c470637141251d4c337 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Wed, 4 Sep 2024 09:02:42 +0000 Subject: [PATCH 03/15] feature : Add the satpy/composites/lightning.py that enables flash_age feature --- satpy/composites/lightning.py | 99 +++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 satpy/composites/lightning.py diff --git a/satpy/composites/lightning.py b/satpy/composites/lightning.py new file mode 100644 index 0000000000..d9c63bbdae --- /dev/null +++ b/satpy/composites/lightning.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Composite classes for the LI instrument.""" + +import logging + +import numpy as np +import xarray as xr + +from satpy.composites import CompositeBase + +LOG = logging.getLogger(__name__) + + +class LightningTimeCompositor(CompositeBase): + """Class used to create the flash_age compositor usefull for lighting event visualisation. + + The datas used are dates related to the lightning event that should be normalised between + 0 and 1. The value 1 corresponds to the latest lightning event and the value 0 corresponds + to the latest lightning event - time_range. The time_range is defined in the satpy/etc/composites/li.yaml + and is in minutes. + """ + def __init__(self, name, prerequisites=None, optional_prerequisites=None, **kwargs): + """Initialisation of the class.""" + self.name = name + super().__init__(name, prerequisites, optional_prerequisites, **kwargs) + # Get the time_range which is in minute + self.time_range = self.attrs["time_range"] + self.standard_name = self.attrs["standard_name"] + + def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: + """Normalised the time in the range between [end_time,end_time - time_range]. + + The range of the normalised data is between 0 and 1 where 0 corresponds to the date end_time - time_range + and 1 to the end_time. Where end_times represent the latest lightning event and time_range is the range of + time you want to see the event.The dates that are earlier to end_time - time_range are removed. + + Args: + data (xr.DataArray): datas containing dates to be normalised + attrs (dict): Attributes suited to the flash_age composite + + Returns: + xr.DataArray: Normalised time + """ + # Compute the maximum time value + end_time = data.max().values + # Compute the minimum time value based on the time range + begin_time = end_time - np.timedelta64(self.time_range, "m") + # Drop values that are bellow begin_time + data = data.where(data > begin_time, drop=True) + # Normalize the time values + normalized_data = (data - begin_time) / (end_time - begin_time) + # Ensure the result is still an xarray.DataArray + return xr.DataArray(normalized_data, dims=data.dims, coords=data.coords,attrs=attrs) + + + @staticmethod + def _update_missing_metadata(existing_attrs, new_attrs): + for key, val in new_attrs.items(): + if key not in existing_attrs and val is not None: + existing_attrs[key] = val + + def _redefine_metadata(self,attrs:dict)->dict: + """Modify the standard_name and name metadatas. + + Args: + attrs (dict): data's attributes + + Returns: + dict: atualised attributes + """ + attrs["name"] = self.standard_name + attrs["standard_name"] =self.standard_name + # Attributes to describe the values range + return attrs + + + def __call__(self,projectables, nonprojectables=None, **attrs): + """Normalise the dates.""" + data = projectables[0] + new_attrs = data.attrs.copy() + self._update_missing_metadata(new_attrs, attrs) + new_attrs = self._redefine_metadata(new_attrs) + return self._normalize_time(data,new_attrs) From efe81f2fe1956a1dca074ba7da9b24b695fea5db Mon Sep 17 00:00:00 2001 From: clement laplace Date: Wed, 4 Sep 2024 11:35:46 +0000 Subject: [PATCH 04/15] modify : use the end_time attributes instead of the maximum values --- satpy/composites/lightning.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/composites/lightning.py b/satpy/composites/lightning.py index d9c63bbdae..b11a791abb 100644 --- a/satpy/composites/lightning.py +++ b/satpy/composites/lightning.py @@ -42,6 +42,7 @@ def __init__(self, name, prerequisites=None, optional_prerequisites=None, **kwar # Get the time_range which is in minute self.time_range = self.attrs["time_range"] self.standard_name = self.attrs["standard_name"] + self.reference_time = self.attrs["reference_time"] def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: """Normalised the time in the range between [end_time,end_time - time_range]. @@ -58,7 +59,7 @@ def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: xr.DataArray: Normalised time """ # Compute the maximum time value - end_time = data.max().values + end_time = np.array(np.datetime64(data.attrs[self.reference_time])) # Compute the minimum time value based on the time range begin_time = end_time - np.timedelta64(self.time_range, "m") # Drop values that are bellow begin_time From 6b59245e1afb4e421b2b9c28e1eaf0d01997ba01 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Fri, 20 Sep 2024 15:10:50 +0000 Subject: [PATCH 05/15] test : Add the test related to the flash_age compositor --- satpy/composites/lightning.py | 3 +- .../tests/compositor_tests/test_lightning.py | 59 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 satpy/tests/compositor_tests/test_lightning.py diff --git a/satpy/composites/lightning.py b/satpy/composites/lightning.py index b11a791abb..af1a6ff02c 100644 --- a/satpy/composites/lightning.py +++ b/satpy/composites/lightning.py @@ -44,6 +44,7 @@ def __init__(self, name, prerequisites=None, optional_prerequisites=None, **kwar self.standard_name = self.attrs["standard_name"] self.reference_time = self.attrs["reference_time"] + def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: """Normalised the time in the range between [end_time,end_time - time_range]. @@ -63,7 +64,7 @@ def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: # Compute the minimum time value based on the time range begin_time = end_time - np.timedelta64(self.time_range, "m") # Drop values that are bellow begin_time - data = data.where(data > begin_time, drop=True) + data = data.where(data >= begin_time, drop=True) # Normalize the time values normalized_data = (data - begin_time) / (end_time - begin_time) # Ensure the result is still an xarray.DataArray diff --git a/satpy/tests/compositor_tests/test_lightning.py b/satpy/tests/compositor_tests/test_lightning.py new file mode 100644 index 0000000000..b686ff915e --- /dev/null +++ b/satpy/tests/compositor_tests/test_lightning.py @@ -0,0 +1,59 @@ +"""Test the flash age compositor.""" +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . + + +import datetime + +import numpy as np +import xarray as xr + +from satpy.composites.lightning import LightningTimeCompositor + + +def test_flash_age_compositor(): + """Test the flash_age compsitor by comparing two xarrays object.""" + comp = LightningTimeCompositor("flash_age",prerequisites=["flash_time"], + standard_name="ligtning_time", + time_range=60, + reference_time="end_time") + attrs_flash_age = {"variable_name": "flash_time","name": "flash_time", + "start_time": datetime.datetime(2024, 8, 1, 10, 50, 0), + "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0),"reader": "li_l2_nc"} + flash_age_value = np.array(["2024-08-01T09:00:00", + "2024-08-01T10:00:00", "2024-08-01T10:30:00","2024-08-01T11:00:00"], dtype="datetime64[ns]") + #Coordinates data (assuming you have longitude and latitude arrays) + flash_age = xr.DataArray( + flash_age_value, + dims=["y"], + coords={ + "crs": "8B +proj=longlat +ellps=WGS84 +type=crs" + },attrs = attrs_flash_age,name="flash_time") + res = comp([flash_age]) + expected_attrs = {"variable_name": "flash_time","name": "lightning_time", + "start_time": datetime.datetime(2024, 8, 1, 10, 50, 0), + "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0),"reader": "li_l2_nc", + "standard_name": "ligtning_time" + } + expected_array = xr.DataArray( + np.array([0.0,0.5,1.0]), + dims=["y"], + coords={ + "crs": "8B +proj=longlat +ellps=WGS84 +type=crs" + },attrs = expected_attrs,name="flash_time") + xr.testing.assert_equal(res,expected_array) From 0ebb1fe2eb6f032ae7272ed9835b626d0a91eda6 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Fri, 20 Sep 2024 15:18:56 +0000 Subject: [PATCH 06/15] feat : Handles case where xarray data is empty for flash_age composites. --- satpy/composites/lightning.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/satpy/composites/lightning.py b/satpy/composites/lightning.py index af1a6ff02c..a7931027b9 100644 --- a/satpy/composites/lightning.py +++ b/satpy/composites/lightning.py @@ -18,6 +18,7 @@ """Composite classes for the LI instrument.""" import logging +import sys import numpy as np import xarray as xr @@ -65,6 +66,10 @@ def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: begin_time = end_time - np.timedelta64(self.time_range, "m") # Drop values that are bellow begin_time data = data.where(data >= begin_time, drop=True) + # exit if data is empty afer filtering + if data.size == 0 : + LOG.error(f"All the flash_age events happened before {begin_time}") + sys.exit(1) # Normalize the time values normalized_data = (data - begin_time) / (end_time - begin_time) # Ensure the result is still an xarray.DataArray From 27eac75d2676442e2c197a6585281fe895efd3e8 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Fri, 20 Sep 2024 15:29:51 +0000 Subject: [PATCH 07/15] typo : Correct operator multi lines error in test_lightning.py file --- satpy/tests/compositor_tests/test_lightning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/compositor_tests/test_lightning.py b/satpy/tests/compositor_tests/test_lightning.py index b686ff915e..990d572fe7 100644 --- a/satpy/tests/compositor_tests/test_lightning.py +++ b/satpy/tests/compositor_tests/test_lightning.py @@ -45,7 +45,7 @@ def test_flash_age_compositor(): "crs": "8B +proj=longlat +ellps=WGS84 +type=crs" },attrs = attrs_flash_age,name="flash_time") res = comp([flash_age]) - expected_attrs = {"variable_name": "flash_time","name": "lightning_time", + expected_attrs = {"variable_name": "flash_time","name": "lightning_time", "start_time": datetime.datetime(2024, 8, 1, 10, 50, 0), "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0),"reader": "li_l2_nc", "standard_name": "ligtning_time" From aae6b98e04a33a40f4d4ad4d243b6d89c2070ac9 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Fri, 20 Sep 2024 15:53:20 +0000 Subject: [PATCH 08/15] test : Add test related to empty array after being filtered --- .../tests/compositor_tests/test_lightning.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/satpy/tests/compositor_tests/test_lightning.py b/satpy/tests/compositor_tests/test_lightning.py index 990d572fe7..25ba3f8c08 100644 --- a/satpy/tests/compositor_tests/test_lightning.py +++ b/satpy/tests/compositor_tests/test_lightning.py @@ -19,6 +19,8 @@ import datetime +import logging +from unittest import mock import numpy as np import xarray as xr @@ -57,3 +59,28 @@ def test_flash_age_compositor(): "crs": "8B +proj=longlat +ellps=WGS84 +type=crs" },attrs = expected_attrs,name="flash_time") xr.testing.assert_equal(res,expected_array) + +def test_empty_array_error(caplog): + """Test when the filtered array is empty.""" + comp = LightningTimeCompositor("flash_age",prerequisites=["flash_time"], + standard_name="ligtning_time", + time_range=60, + reference_time="end_time") + attrs_flash_age = {"variable_name": "flash_time","name": "flash_time", + "start_time": datetime.datetime(2024, 8, 1, 10, 50, 0), + "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0),"reader": "li_l2_nc"} + flash_age_value = np.array(["2024-08-01T09:00:00"], dtype="datetime64[ns]") + flash_age = xr.DataArray( + flash_age_value, + dims=["y"], + coords={ + "crs": "8B +proj=longlat +ellps=WGS84 +type=crs" + },attrs = attrs_flash_age,name="flash_time") + with mock.patch("sys.exit") as mock_exit: + # Capture logging output + with caplog.at_level(logging.ERROR): + _ = comp([flash_age]) + + mock_exit.assert_called_once_with(1) + + assert "All the flash_age events happened before 2024-08-01T10:00:00" in caplog.text From 589743172c540d233d97eece227e7f5cdeb86f0b Mon Sep 17 00:00:00 2001 From: clement laplace Date: Fri, 20 Sep 2024 16:23:02 +0000 Subject: [PATCH 09/15] test : Add test concerning the missing data --- .../tests/compositor_tests/test_lightning.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/satpy/tests/compositor_tests/test_lightning.py b/satpy/tests/compositor_tests/test_lightning.py index 25ba3f8c08..5cd024401e 100644 --- a/satpy/tests/compositor_tests/test_lightning.py +++ b/satpy/tests/compositor_tests/test_lightning.py @@ -84,3 +84,31 @@ def test_empty_array_error(caplog): mock_exit.assert_called_once_with(1) assert "All the flash_age events happened before 2024-08-01T10:00:00" in caplog.text + +def test_update_missing_metadata(): + """Test the _update_missing_metadata method.""" + existing_attrs = { + "standard_name": "lightning_event_time", + "time_range": 30 + } + + # New metadata to be merged + new_attrs = { + "standard_name": None, # Should not overwrite since it's None + "reference_time": "2023-09-20T00:00:00Z", # Should be added + "units": "seconds" # Should be added + } + + # Expected result after merging + expected_attrs = { + "standard_name": "lightning_event_time", # Should remain the same + "time_range": 30, # Should remain the same + "reference_time": "2023-09-20T00:00:00Z", # Should be added + "units": "seconds" # Should be added + } + + # Call the static method + LightningTimeCompositor._update_missing_metadata(existing_attrs, new_attrs) + + # Assert the final state of existing_attrs is as expected + assert existing_attrs == expected_attrs From 3edf89391cfc6a29fed11baac9fe7c529b4cca8a Mon Sep 17 00:00:00 2001 From: clement laplace Date: Mon, 23 Sep 2024 08:04:48 +0000 Subject: [PATCH 10/15] typo : Erase useless comment --- satpy/tests/compositor_tests/test_lightning.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/compositor_tests/test_lightning.py b/satpy/tests/compositor_tests/test_lightning.py index 5cd024401e..4c1f8b9a8c 100644 --- a/satpy/tests/compositor_tests/test_lightning.py +++ b/satpy/tests/compositor_tests/test_lightning.py @@ -39,7 +39,6 @@ def test_flash_age_compositor(): "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0),"reader": "li_l2_nc"} flash_age_value = np.array(["2024-08-01T09:00:00", "2024-08-01T10:00:00", "2024-08-01T10:30:00","2024-08-01T11:00:00"], dtype="datetime64[ns]") - #Coordinates data (assuming you have longitude and latitude arrays) flash_age = xr.DataArray( flash_age_value, dims=["y"], From a008f9f1914f326cfe21842c1bd5fbce8242ddd0 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Fri, 10 Jan 2025 10:47:13 +0000 Subject: [PATCH 11/15] correction : Apllied corrections asked in this MR https://github.com/pytroll/satpy/pull/2895 --- satpy/composites/lightning.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/satpy/composites/lightning.py b/satpy/composites/lightning.py index a7931027b9..4f72ffd7a1 100644 --- a/satpy/composites/lightning.py +++ b/satpy/composites/lightning.py @@ -18,7 +18,6 @@ """Composite classes for the LI instrument.""" import logging -import sys import numpy as np import xarray as xr @@ -38,16 +37,15 @@ class LightningTimeCompositor(CompositeBase): """ def __init__(self, name, prerequisites=None, optional_prerequisites=None, **kwargs): """Initialisation of the class.""" - self.name = name super().__init__(name, prerequisites, optional_prerequisites, **kwargs) # Get the time_range which is in minute self.time_range = self.attrs["time_range"] self.standard_name = self.attrs["standard_name"] - self.reference_time = self.attrs["reference_time"] + self.reference_time_attr = self.attrs["reference_time"] - def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: - """Normalised the time in the range between [end_time,end_time - time_range]. + def _normalize_time(self, data:xr.DataArray, attrs:dict) -> xr.DataArray: + """Normalize the time in the range between [end_time, end_time - time_range]. The range of the normalised data is between 0 and 1 where 0 corresponds to the date end_time - time_range and 1 to the end_time. Where end_times represent the latest lightning event and time_range is the range of @@ -61,7 +59,7 @@ def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: xr.DataArray: Normalised time """ # Compute the maximum time value - end_time = np.array(np.datetime64(data.attrs[self.reference_time])) + end_time = np.array(np.datetime64(data.attrs[self.reference_time_attr])) # Compute the minimum time value based on the time range begin_time = end_time - np.timedelta64(self.time_range, "m") # Drop values that are bellow begin_time @@ -69,11 +67,13 @@ def _normalize_time(self,data:xr.DataArray,attrs:dict)->xr.DataArray: # exit if data is empty afer filtering if data.size == 0 : LOG.error(f"All the flash_age events happened before {begin_time}") - sys.exit(1) + raise ValueError(f"Invalid data: data size is zero. All flash_age \ + events occurred before the specified start time ({begin_time})." + ) # Normalize the time values normalized_data = (data - begin_time) / (end_time - begin_time) # Ensure the result is still an xarray.DataArray - return xr.DataArray(normalized_data, dims=data.dims, coords=data.coords,attrs=attrs) + return xr.DataArray(normalized_data, dims=data.dims, coords=data.coords, attrs=attrs) @staticmethod @@ -92,7 +92,7 @@ def _redefine_metadata(self,attrs:dict)->dict: dict: atualised attributes """ attrs["name"] = self.standard_name - attrs["standard_name"] =self.standard_name + attrs["standard_name"] = self.standard_name # Attributes to describe the values range return attrs @@ -103,4 +103,4 @@ def __call__(self,projectables, nonprojectables=None, **attrs): new_attrs = data.attrs.copy() self._update_missing_metadata(new_attrs, attrs) new_attrs = self._redefine_metadata(new_attrs) - return self._normalize_time(data,new_attrs) + return self._normalize_time(data, new_attrs) From 7676013dc8ad35fcf25abb3491ec022032a5a087 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Tue, 14 Jan 2025 10:05:40 +0000 Subject: [PATCH 12/15] test: Correct the test_lightning.py::test_empty_array_error --- satpy/composites/lightning.py | 4 ++-- .../tests/compositor_tests/test_lightning.py | 22 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/satpy/composites/lightning.py b/satpy/composites/lightning.py index 4f72ffd7a1..be4d0c769b 100644 --- a/satpy/composites/lightning.py +++ b/satpy/composites/lightning.py @@ -67,8 +67,8 @@ def _normalize_time(self, data:xr.DataArray, attrs:dict) -> xr.DataArray: # exit if data is empty afer filtering if data.size == 0 : LOG.error(f"All the flash_age events happened before {begin_time}") - raise ValueError(f"Invalid data: data size is zero. All flash_age \ - events occurred before the specified start time ({begin_time})." + raise ValueError(f"Invalid data: data size is zero. All flash_age " + f"events occurred before the specified start time ({begin_time})." ) # Normalize the time values normalized_data = (data - begin_time) / (end_time - begin_time) diff --git a/satpy/tests/compositor_tests/test_lightning.py b/satpy/tests/compositor_tests/test_lightning.py index 4c1f8b9a8c..6b1dab7b4c 100644 --- a/satpy/tests/compositor_tests/test_lightning.py +++ b/satpy/tests/compositor_tests/test_lightning.py @@ -20,9 +20,9 @@ import datetime import logging -from unittest import mock import numpy as np +import pytest import xarray as xr from satpy.composites.lightning import LightningTimeCompositor @@ -66,8 +66,9 @@ def test_empty_array_error(caplog): time_range=60, reference_time="end_time") attrs_flash_age = {"variable_name": "flash_time","name": "flash_time", - "start_time": datetime.datetime(2024, 8, 1, 10, 50, 0), - "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0),"reader": "li_l2_nc"} + "start_time": np.datetime64(datetime.datetime(2024, 8, 1, 10, 0, 0)), + "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0), + "reader": "li_l2_nc"} flash_age_value = np.array(["2024-08-01T09:00:00"], dtype="datetime64[ns]") flash_age = xr.DataArray( flash_age_value, @@ -75,14 +76,17 @@ def test_empty_array_error(caplog): coords={ "crs": "8B +proj=longlat +ellps=WGS84 +type=crs" },attrs = attrs_flash_age,name="flash_time") - with mock.patch("sys.exit") as mock_exit: - # Capture logging output - with caplog.at_level(logging.ERROR): + with caplog.at_level(logging.ERROR): + # Simulate the operation that raises the exception + with pytest.raises(ValueError, match="data size is zero") as excinfo: _ = comp([flash_age]) - mock_exit.assert_called_once_with(1) - - assert "All the flash_age events happened before 2024-08-01T10:00:00" in caplog.text + # Assert the exception message + assert str(excinfo.value) == ( + f"Invalid data: data size is zero. All flash_age events occurred before " + f"the specified start time ({attrs_flash_age['start_time']})." + ) + assert "All the flash_age events happened before 2024-08-01T10:00:00" in caplog.text def test_update_missing_metadata(): """Test the _update_missing_metadata method.""" From f202054eb07c9c2935a9836196a5f089dff6ff24 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Tue, 21 Jan 2025 12:15:22 +0000 Subject: [PATCH 13/15] fix : fix the handling to drop date with dask array that is introduced with the PR #2985 --- satpy/composites/lightning.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/satpy/composites/lightning.py b/satpy/composites/lightning.py index be4d0c769b..5fc52d7c9c 100644 --- a/satpy/composites/lightning.py +++ b/satpy/composites/lightning.py @@ -63,7 +63,9 @@ def _normalize_time(self, data:xr.DataArray, attrs:dict) -> xr.DataArray: # Compute the minimum time value based on the time range begin_time = end_time - np.timedelta64(self.time_range, "m") # Drop values that are bellow begin_time - data = data.where(data >= begin_time, drop=True) + condition_time = data >= begin_time + condition_time_computed = condition_time.compute() + data = data.where(condition_time_computed, drop=True) # exit if data is empty afer filtering if data.size == 0 : LOG.error(f"All the flash_age events happened before {begin_time}") From 4c25948f3ea474780142088157076e3e0a85c8d6 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Tue, 21 Jan 2025 12:50:25 +0000 Subject: [PATCH 14/15] fix: Correct the test by replacing numpy array to dask array to simulate the real case data --- satpy/tests/compositor_tests/test_lightning.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/satpy/tests/compositor_tests/test_lightning.py b/satpy/tests/compositor_tests/test_lightning.py index 6b1dab7b4c..52eea22f3b 100644 --- a/satpy/tests/compositor_tests/test_lightning.py +++ b/satpy/tests/compositor_tests/test_lightning.py @@ -21,6 +21,7 @@ import datetime import logging +import dask.array as da import numpy as np import pytest import xarray as xr @@ -37,7 +38,7 @@ def test_flash_age_compositor(): attrs_flash_age = {"variable_name": "flash_time","name": "flash_time", "start_time": datetime.datetime(2024, 8, 1, 10, 50, 0), "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0),"reader": "li_l2_nc"} - flash_age_value = np.array(["2024-08-01T09:00:00", + flash_age_value = da.array(["2024-08-01T09:00:00", "2024-08-01T10:00:00", "2024-08-01T10:30:00","2024-08-01T11:00:00"], dtype="datetime64[ns]") flash_age = xr.DataArray( flash_age_value, @@ -52,7 +53,7 @@ def test_flash_age_compositor(): "standard_name": "ligtning_time" } expected_array = xr.DataArray( - np.array([0.0,0.5,1.0]), + da.array([0.0,0.5,1.0]), dims=["y"], coords={ "crs": "8B +proj=longlat +ellps=WGS84 +type=crs" @@ -69,7 +70,7 @@ def test_empty_array_error(caplog): "start_time": np.datetime64(datetime.datetime(2024, 8, 1, 10, 0, 0)), "end_time": datetime.datetime(2024, 8, 1, 11, 0, 0), "reader": "li_l2_nc"} - flash_age_value = np.array(["2024-08-01T09:00:00"], dtype="datetime64[ns]") + flash_age_value = da.array(["2024-08-01T09:00:00"], dtype="datetime64[ns]") flash_age = xr.DataArray( flash_age_value, dims=["y"], From 01237e2cbd0098931240e40cd4f7f5eeda3e532f Mon Sep 17 00:00:00 2001 From: clement laplace Date: Wed, 22 Jan 2025 06:58:12 +0000 Subject: [PATCH 15/15] feat: Add the component element true_color_with_night_ir105_flash_age --- satpy/etc/composites/fci.yaml | 149 +++++++++++++++++---------------- satpy/etc/enhancements/li.yaml | 2 +- 2 files changed, 79 insertions(+), 72 deletions(-) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 5dd812c73f..07ff48da8a 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -1,17 +1,17 @@ sensor_name: visir/fci composites: -### L2 + ### L2 binary_cloud_mask: # This will set all clear pixels to '0', all pixels with cloudy features (meteorological/dust/ash clouds) to '1' and # missing/undefined pixels to 'nan'. This can be used for the official EUMETSAT cloud mask product (CLM). compositor: !!python/name:satpy.composites.CategoricalDataCompositor prerequisites: - - name: 'cloud_state' - lut: [ .nan, 0, 1, 1, 1, 1, 1, 1, 0, .nan ] + - name: "cloud_state" + lut: [.nan, 0, 1, 1, 1, 1, 1, 1, 0, .nan] standard_name: binary_cloud_mask -### Night Layers + ### Night Layers night_ir105: compositor: !!python/name:satpy.composites.SingleBandCompositor prerequisites: @@ -41,7 +41,7 @@ composites: - night_ir_alpha - _night_background_hires -### Green Corrections + ### Green Corrections ndvi_hybrid_green: description: > The FCI green band at 0.51 µm deliberately misses the chlorophyll band, such that @@ -58,7 +58,7 @@ composites: - name: vis_06 modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced] - name: vis_08 - modifiers: [sunz_corrected, sunz_reduced ] + modifiers: [sunz_corrected, sunz_reduced] standard_name: toa_bidirectional_reflectance ndvi_hybrid_green_raw: @@ -76,18 +76,18 @@ composites: ndvi_hybrid_green_fully_sunzencorrected: description: Same as ndvi_hybrid_green, but without Sun-zenith reduction compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen - limits: [ 0.15, 0.05 ] + limits: [0.15, 0.05] strength: 3.0 prerequisites: - name: vis_05 - modifiers: [ sunz_corrected, rayleigh_corrected ] + modifiers: [sunz_corrected, rayleigh_corrected] - name: vis_06 - modifiers: [ sunz_corrected, rayleigh_corrected ] + modifiers: [sunz_corrected, rayleigh_corrected] - name: vis_08 - modifiers: [ sunz_corrected ] + modifiers: [sunz_corrected] standard_name: toa_bidirectional_reflectance -### True Color + ### True Color true_color: compositor: !!python/name:satpy.composites.SelfSharpenedRGB description: > @@ -190,7 +190,7 @@ composites: - name: vis_04 standard_name: true_color_reproduction_color_stretch -### True Color with LI lightning + ### True Color with LI lightning true_color_with_night_ir105_acc_flash: compositor: !!python/name:satpy.composites.BackgroundCompositor @@ -227,74 +227,81 @@ composites: - group_radiance_alpha - true_color_with_night_ir105 -### GeoColor + true_color_with_night_ir105_flash_age: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: imager_with_lightning + prerequisites: + - flash_age + - true_color_with_night_ir105 + + ### GeoColor geo_color: - compositor: !!python/name:satpy.composites.DayNightCompositor - description: > - GeoColor is a multi-layer blended RGB composite where the day-time part of the image is represented by true - color imagery and the nighttime part of the image by a three layer vertically blended stack composed of a - high-level cloud layer (single IR window channel), a low-level cloud layer (IR split window) and a static - surface terrain layer with city lights (NASA Black Marble). - references: - Research Article: https://journals.ametsoc.org/view/journals/atot/37/3/JTECH-D-19-0134.1.xml - lim_low: 78 - lim_high: 88 - standard_name: geo_color_day_night_blend - prerequisites: - - true_color - - geo_color_night + compositor: !!python/name:satpy.composites.DayNightCompositor + description: > + GeoColor is a multi-layer blended RGB composite where the day-time part of the image is represented by true + color imagery and the nighttime part of the image by a three layer vertically blended stack composed of a + high-level cloud layer (single IR window channel), a low-level cloud layer (IR split window) and a static + surface terrain layer with city lights (NASA Black Marble). + references: + Research Article: https://journals.ametsoc.org/view/journals/atot/37/3/JTECH-D-19-0134.1.xml + lim_low: 78 + lim_high: 88 + standard_name: geo_color_day_night_blend + prerequisites: + - true_color + - geo_color_night geo_color_high_clouds: - standard_name: geo_color_high_clouds - compositor: !!python/name:satpy.composites.HighCloudCompositor - prerequisites: - - name: ir_105 + standard_name: geo_color_high_clouds + compositor: !!python/name:satpy.composites.HighCloudCompositor + prerequisites: + - name: ir_105 geo_color_low_clouds: - standard_name: geo_color_low_clouds - compositor: !!python/name:satpy.composites.LowCloudCompositor - values_water: 0 - values_land: 100 - range_water: [0.0, 4.0] - range_land: [1.5, 4.0] - prerequisites: - - compositor: !!python/name:satpy.composites.DifferenceCompositor - prerequisites: - - name: ir_105 - - name: ir_38 - - name: ir_105 - - compositor: !!python/name:satpy.composites.StaticImageCompositor - standard_name: land_water_mask - url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" - known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" + standard_name: geo_color_low_clouds + compositor: !!python/name:satpy.composites.LowCloudCompositor + values_water: 0 + values_land: 100 + range_water: [0.0, 4.0] + range_land: [1.5, 4.0] + prerequisites: + - compositor: !!python/name:satpy.composites.DifferenceCompositor + prerequisites: + - name: ir_105 + - name: ir_38 + - name: ir_105 + - compositor: !!python/name:satpy.composites.StaticImageCompositor + standard_name: land_water_mask + url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" + known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" geo_color_background_with_low_clouds: - compositor: !!python/name:satpy.composites.BackgroundCompositor - standard_name: night_ir_with_background - prerequisites: - - geo_color_low_clouds - - _night_background_hires + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_low_clouds + - _night_background_hires geo_color_night: - compositor: !!python/name:satpy.composites.BackgroundCompositor - standard_name: night_ir_with_background - prerequisites: - - geo_color_high_clouds - - geo_color_background_with_low_clouds + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_high_clouds + - geo_color_background_with_low_clouds -### IR-Sandwich + ### IR-Sandwich ir_sandwich: compositor: !!python/name:satpy.composites.SandwichCompositor standard_name: ir_sandwich prerequisites: - - name: 'vis_06' - modifiers: [ sunz_corrected ] + - name: "vis_06" + modifiers: [sunz_corrected] - name: colorized_ir_clouds colorized_ir_clouds: compositor: !!python/name:satpy.composites.SingleBandCompositor prerequisites: - - name: 'ir_105' + - name: "ir_105" standard_name: colorized_ir_clouds ir_sandwich_with_night_colorized_ir_clouds: @@ -306,7 +313,7 @@ composites: - ir_sandwich - colorized_ir_clouds -### other RGBs + ### other RGBs cloud_type: description: > Equal to cimss_cloud_type recipe, but with additional sunz_reducer modifier to avoid saturation at the terminator. @@ -316,11 +323,11 @@ composites: compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: - name: nir_13 - modifiers: [ sunz_corrected, sunz_reduced ] + modifiers: [sunz_corrected, sunz_reduced] - name: vis_06 - modifiers: [ sunz_corrected, sunz_reduced ] + modifiers: [sunz_corrected, sunz_reduced] - name: nir_16 - modifiers: [ sunz_corrected, sunz_reduced ] + modifiers: [sunz_corrected, sunz_reduced] standard_name: cimss_cloud_type cloud_type_with_night_ir105: @@ -416,10 +423,10 @@ composites: Recipe: https://resources.eumetrain.org/RGBguide/recipes/RGB_recipes.pdf compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: - - name: vis_08 - modifiers: [sunz_corrected] - - name: nir_16 - modifiers: [sunz_corrected] - - name: ir_38 - modifiers: [nir_reflectance] + - name: vis_08 + modifiers: [sunz_corrected] + - name: nir_16 + modifiers: [sunz_corrected] + - name: ir_38 + modifiers: [nir_reflectance] standard_name: snow diff --git a/satpy/etc/enhancements/li.yaml b/satpy/etc/enhancements/li.yaml index 9aaa5c4a0b..82b6056b9a 100644 --- a/satpy/etc/enhancements/li.yaml +++ b/satpy/etc/enhancements/li.yaml @@ -81,4 +81,4 @@ enhancements: method: !!python/name:satpy.enhancements.colorize kwargs: palettes: - - { colors: ylorrd, min_value: 0, max_value: 1 } + - { colors: ylorrd, reverse: True, min_value: 0, max_value: 1 }