Earlinet + 3d colocation updates: Include Level1 data and Introduce LayerLimits

pyaerocom/_lowlevel_helpers.py

@@ -712,3 +712,10 @@ class RegridResDeg(TypedDict):
 
     lat_res_deg: float
     lon_res_deg: float
+
+
+class LayerLimits(TypedDict):
+    """Typed dict of 3D colocation layer limits"""
+
+    start: float
+    end: float

pyaerocom/colocation/colocation_3d.py

@@ -14,7 +14,7 @@
 
 from pyaerocom import __version__ as pya_ver
 from pyaerocom import const
-from pyaerocom._lowlevel_helpers import RegridResDeg
+from pyaerocom._lowlevel_helpers import LayerLimits, RegridResDeg
 from pyaerocom.exceptions import (
     DataUnitError,
     DimensionOrderError,
@@ -56,7 +56,7 @@ def _colocate_vertical_profile_gridded(
     colocate_time=False,
     use_climatology_ref=False,
     resample_how=None,
-    layer_limits: dict[str, dict[str, float]] = None,
+    layer_limits: LayerLimits | None = None,
     obs_stat_data=None,
     ungridded_lons=None,
     ungridded_lats=None,
@@ -313,8 +313,8 @@ def colocate_vertical_profile_gridded(
     colocate_time: bool = False,
     use_climatology_ref: bool = False,
     resample_how: str | dict = None,
-    colocation_layer_limits: list[dict] = None,
-    profile_layer_limits: list[dict] = None,
+    colocation_layer_limits: tuple[LayerLimits, ...] | None = None,
+    profile_layer_limits: tuple[LayerLimits, ...] | None = None,
     **kwargs,
 ) -> ColocatedDataLists:
     """
@@ -479,7 +479,7 @@ def colocate_vertical_profile_gridded(
             var_aerocom=var_aerocom,
             var_ref_aerocom=var_ref_aerocom,
         )
-        for layer_limits in [colocation_layer_limits, profile_layer_limits]
+        for layer_limits in (colocation_layer_limits, profile_layer_limits)
     ]
     # Create a namedtuple for output.
     # Each element in the tuple is a list of ColocatedData objects.

pyaerocom/colocation/colocation_setup.py

@@ -16,7 +16,7 @@
 )
 
 from pyaerocom import const
-from pyaerocom._lowlevel_helpers import RegridResDeg
+from pyaerocom._lowlevel_helpers import LayerLimits, RegridResDeg
 from pyaerocom.config import ALL_REGION_NAME
 from pyaerocom.helpers import start_stop
 from pyaerocom.io.pyaro.pyaro_config import PyaroConfig
@@ -376,9 +376,8 @@ def validate_basedirs(cls, v):
     obs_vert_type: str | None = None
     obs_ts_type_read: str | dict | None = None
     obs_filters: dict = {}
-    _obs_is_vertical_profile: bool = False
-    colocation_layer_limits: dict[str, float] | None = None
-    profile_layer_limits: dict | None = None
+    colocation_layer_limits: tuple[LayerLimits, ...] | None = None
+    profile_layer_limits: tuple[LayerLimits, ...] | None = None
     read_opts_ungridded: dict | None = {}
 
     # Attributes related to model data

pyaerocom/colocation/colocation_utils.py

@@ -170,7 +170,7 @@ def colocate_gridded_gridded(
 ):
     """Colocate 2 gridded data objects
 
-    Todo
+    TODO
     ----
     - think about vertical dimension (vert_scheme input not used at the moment)
 

pyaerocom/colocation/colocator.py

@@ -86,6 +86,7 @@ def __init__(self, colocation_setup: ColocationSetup | dict, **kwargs):
 
         self._model_reader: ReadGridded | ReadMscwCtm | ReadCAMS2_83 | None = None
         self._obs_reader: Any | None = None
+        self._obs_is_vertical_profile: bool = False
         self.obs_filters: dict = colocation_setup.obs_filters.copy()
 
     @property
@@ -136,7 +137,7 @@ def obs_is_vertical_profile(self):
         """
         bool: True if obs_id refers to a VerticalProfile, else False
         """
-        return self.colocation_setup._obs_is_vertical_profile
+        return self._obs_is_vertical_profile
 
     @obs_is_vertical_profile.setter
     def obs_is_vertical_profile(self, value):

pyaerocom/io/read_earlinet.py

@@ -24,7 +24,7 @@ class ReadEarlinet(ReadUngriddedBase):
     _FILEMASK = "*.*"
 
     #: version log of this class (for caching)
-    __version__ = "0.16_" + ReadUngriddedBase.__baseversion__
+    __version__ = "0.17_" + ReadUngriddedBase.__baseversion__
 
     #: Name of dataset (OBS_ID)
     DATA_ID = const.EARLINET_NAME
@@ -48,22 +48,12 @@ class ReadEarlinet(ReadUngriddedBase):
     # at an hourly reoslution. Some files are a little less, but typically this is the case
     TS_TYPE = "hourly"
 
-    #: dictionary specifying the file search patterns for each variable
-    # VAR_PATTERNS_FILE = {
-    #     "ec532aer": "*.e532",
-    #     "ec355aer": "*.e355",
-    #     "bsc532aer": "*.b532",
-    #     "bsc355aer": "*.b355",
-    #     "bsc1064aer": "*.b1064",
-    #     "zdust": "*.e*",
-    # }
-
     VAR_PATTERNS_FILE = {
-        "ec532aer": "_Lev02_e0532",
-        "ec355aer": "_Lev02_e0355",
-        "bsc532aer": "_Lev02_b0532",
-        "bsc355aer": "_Lev02_b0355",
-        "bsc1064aer": "_Lev02_b1064",
+        "ec532aer": "_e0532",
+        "ec355aer": "_e0355",
+        "bsc532aer": "_b0532",
+        "bsc355aer": "_b0355",
+        "bsc1064aer": "_b1064",
         # "zdust": "*.e*", # not sure if EARLINET has this anymore
     }
 

tests/colocation/test_colocation_3d.py

@@ -5,6 +5,7 @@
 import pytest
 
 from pyaerocom import GriddedData
+from pyaerocom._lowlevel_helpers import LayerLimits
 from pyaerocom.colocation.colocation_3d import (
     ColocatedDataLists,
     colocate_vertical_profile_gridded,
@@ -70,12 +71,8 @@ def example_earlinet_ungriddeddata():
             "mean",
             {"monthly": {"daily": 25}},
             False,
-            [
-                {"start": 0, "end": 6000},
-            ],
-            [
-                {"start": 0, "end": 6000},
-            ],
+            ({"start": 0, "end": 6000},),
+            ({"start": 0, "end": 6000},),
             id="fake_data",
         )
     ],
@@ -87,8 +84,8 @@ def test_colocate_vertical_profile_gridded(
     resample_how,
     min_num_obs,
     use_climatology_ref,
-    colocation_layer_limits,
-    profile_layer_limits,
+    colocation_layer_limits: tuple[LayerLimits, ...],
+    profile_layer_limits: tuple[LayerLimits, ...],
 ):
     colocated_data_list = colocate_vertical_profile_gridded(
         data=fake_model_data_with_altitude,