update modules to new config

sed/calibrator/delay.py

@@ -55,10 +55,10 @@ def __init__(
         else:
             logger.handlers[0].setLevel(WARNING)
 
-        self.adc_column: str = self._config["dataframe"].get("adc_column", None)
-        self.delay_column: str = self._config["dataframe"]["delay_column"]
-        self.corrected_delay_column = self._config["dataframe"].get(
-            "corrected_delay_column",
+        self.adc_column: str = config["dataframe"]["columns"]["adc"]
+        self.delay_column: str = config["dataframe"]["columns"]["delay"]
+        self.corrected_delay_column = self._config["dataframe"]["columns"].get(
+            "corrected_delay",
             self.delay_column,
         )
         self.calibration: dict[str, Any] = self._config["delay"].get("calibration", {})
@@ -87,9 +87,9 @@ def append_delay_axis(
             df (pd.DataFrame | dask.dataframe.DataFrame): The dataframe where
                 to apply the delay calibration to.
             adc_column (str, optional): Source column for delay calibration.
-                Defaults to config["dataframe"]["adc_column"].
+                Defaults to config["dataframe"]["columns"]["adc"].
             delay_column (str, optional): Destination column for delay calibration.
-                Defaults to config["dataframe"]["delay_column"].
+                Defaults to config["dataframe"]["columns"]["delay"].
             calibration (dict, optional): Calibration dictionary with parameters for
                 delay calibration.
             adc_range (tuple | list | np.ndarray, optional): The range of used

sed/calibrator/energy.py

@@ -111,12 +111,12 @@ def __init__(
         self.peaks: np.ndarray = np.asarray([])
         self.calibration: dict[str, Any] = self._config["energy"].get("calibration", {})
 
-        self.tof_column = self._config["dataframe"]["tof_column"]
-        self.tof_ns_column = self._config["dataframe"].get("tof_ns_column", None)
-        self.corrected_tof_column = self._config["dataframe"]["corrected_tof_column"]
-        self.energy_column = self._config["dataframe"]["energy_column"]
-        self.x_column = self._config["dataframe"]["x_column"]
-        self.y_column = self._config["dataframe"]["y_column"]
+        self.tof_column = self._config["dataframe"]["columns"]["tof"]
+        self.tof_ns_column = self._config["dataframe"]["columns"].get("tof_ns", None)
+        self.corrected_tof_column = self._config["dataframe"]["columns"]["corrected_tof"]
+        self.energy_column = self._config["dataframe"]["columns"]["energy"]
+        self.x_column = self._config["dataframe"]["columns"]["x"]
+        self.y_column = self._config["dataframe"]["columns"]["y"]
         self.binwidth: float = self._config["dataframe"]["tof_binwidth"]
         self.binning: int = self._config["dataframe"]["tof_binning"]
         self.x_width = self._config["energy"]["x_width"]
@@ -125,7 +125,7 @@ def __init__(
         self.tof_fermi = self._config["energy"]["tof_fermi"] / self.binning
         self.color_clip = self._config["energy"]["color_clip"]
         self.sector_delays = self._config["dataframe"].get("sector_delays", None)
-        self.sector_id_column = self._config["dataframe"].get("sector_id_column", None)
+        self.sector_id_column = self._config["dataframe"]["columns"].get("sector_id", None)
         self.offsets: dict[str, Any] = self._config["energy"].get("offsets", {})
         self.correction: dict[str, Any] = self._config["energy"].get("correction", {})
 
@@ -202,7 +202,7 @@ def bin_data(
         Args:
             data_files (list[str]): list of file names to bin
             axes (list[str], optional): bin axes. Defaults to
-                config["dataframe"]["tof_column"].
+                config["dataframe"]["columns"]["tof"].
             bins (list[int], optional): number of bins.
                 Defaults to config["energy"]["bins"].
             ranges (Sequence[tuple[float, float]], optional): bin ranges.
@@ -787,9 +787,9 @@ def append_energy_axis(
             df (pd.DataFrame | dask.dataframe.DataFrame):
                 Dataframe to apply the energy axis calibration to.
             tof_column (str, optional): Label of the source column.
-                Defaults to config["dataframe"]["tof_column"].
+                Defaults to config["dataframe"]["columns"]["tof"].
             energy_column (str, optional): Label of the destination column.
-                Defaults to config["dataframe"]["energy_column"].
+                Defaults to config["dataframe"]["columns"]["energy"].
             calibration (dict, optional): Calibration dictionary. If provided,
                 overrides calibration from class or config.
                 Defaults to self.calibration or config["energy"]["calibration"].
@@ -933,9 +933,9 @@ def append_tof_ns_axis(
         Args:
             df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to convert.
             tof_column (str, optional): Name of the column containing the
-                time-of-flight steps. Defaults to config["dataframe"]["tof_column"].
+                time-of-flight steps. Defaults to config["dataframe"]["columns"]["tof"].
             tof_ns_column (str, optional): Name of the column to store the
-                time-of-flight in nanoseconds. Defaults to config["dataframe"]["tof_ns_column"].
+                time-of-flight in nanoseconds. Defaults to config["dataframe"]["columns"]["tof_ns"].
             binwidth (float, optional): Time-of-flight binwidth in ns.
                 Defaults to config["energy"]["tof_binwidth"].
             binning (int, optional): Time-of-flight binning factor.
@@ -1364,9 +1364,9 @@ def apply_energy_correction(
             df (pd.DataFrame | dask.dataframe.DataFrame): The dataframe where
                 to apply the energy correction to.
             tof_column (str, optional): Name of the source column to convert.
-                Defaults to config["dataframe"]["tof_column"].
+                Defaults to config["dataframe"]["columns"]["tof"].
             new_tof_column (str, optional): Name of the destination column to convert.
-                Defaults to config["dataframe"]["corrected_tof_column"].
+                Defaults to config["dataframe"]["columns"]["corrected_tof"].
             correction_type (str, optional): Type of correction to apply to the TOF
                 axis. Valid values are:
 
@@ -1477,9 +1477,9 @@ def align_dld_sectors(
         Args:
             df (dask.dataframe.DataFrame): Dataframe to use.
             tof_column (str, optional): Name of the column containing the time-of-flight values.
-                Defaults to config["dataframe"]["tof_column"].
+                Defaults to config["dataframe"]["columns"]["tof"].
             sector_id_column (str, optional): Name of the column containing the sector id values.
-                Defaults to config["dataframe"]["sector_id_column"].
+                Defaults to config["dataframe"]["columns"]["sector_id"].
             sector_delays (np.ndarray, optional): Array containing the sector delays. Defaults to
                 config["dataframe"]["sector_delays"].
 

sed/calibrator/momentum.py

@@ -127,12 +127,12 @@ def __init__(
         self.adjust_params: dict[str, Any] = {}
         self.calibration: dict[str, Any] = self._config["momentum"].get("calibration", {})
 
-        self.x_column = self._config["dataframe"]["x_column"]
-        self.y_column = self._config["dataframe"]["y_column"]
-        self.corrected_x_column = self._config["dataframe"]["corrected_x_column"]
-        self.corrected_y_column = self._config["dataframe"]["corrected_y_column"]
-        self.kx_column = self._config["dataframe"]["kx_column"]
-        self.ky_column = self._config["dataframe"]["ky_column"]
+        self.x_column = self._config["dataframe"]["columns"]["x"]
+        self.y_column = self._config["dataframe"]["columns"]["y"]
+        self.corrected_x_column = self._config["dataframe"]["columns"]["corrected_x"]
+        self.corrected_y_column = self._config["dataframe"]["columns"]["corrected_y"]
+        self.kx_column = self._config["dataframe"]["columns"]["kx"]
+        self.ky_column = self._config["dataframe"]["columns"]["ky"]
 
         self._state: int = 0
 
@@ -1734,15 +1734,15 @@ def apply_corrections(
             df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to apply
                 the distortion correction to.
             x_column (str, optional): Label of the 'X' column before momentum
-                distortion correction. Defaults to config["momentum"]["x_column"].
+                distortion correction. Defaults to config["dataframe"]["columns"]["x"].
             y_column (str, optional): Label of the 'Y' column before momentum
-                distortion correction. Defaults to config["momentum"]["y_column"].
+                distortion correction. Defaults to config["dataframe"]["columns"]["y"].
             new_x_column (str, optional): Label of the 'X' column after momentum
                 distortion correction.
-                Defaults to config["momentum"]["corrected_x_column"].
+                Defaults to config["dataframe"]["columns"]["corrected_x"].
             new_y_column (str, optional): Label of the 'Y' column after momentum
                 distortion correction.
-                Defaults to config["momentum"]["corrected_y_column"].
+                Defaults to config["dataframe"]["columns"]["corrected_y"].
 
         Returns:
             tuple[pd.DataFrame | dask.dataframe.DataFrame, dict]: Dataframe with
@@ -1883,15 +1883,15 @@ def append_k_axis(
             df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to apply the
                 distortion correction to.
             x_column (str, optional): Label of the source 'X' column.
-                Defaults to config["momentum"]["corrected_x_column"] or
-                config["momentum"]["x_column"] (whichever is present).
+                Defaults to config["dataframe"]["columns"]["corrected_x"] or
+                config["dataframe"]["columns"]["x"] (whichever is present).
             y_column (str, optional): Label of the source 'Y' column.
-                Defaults to config["momentum"]["corrected_y_column"] or
-                config["momentum"]["y_column"] (whichever is present).
+                Defaults to config["dataframe"]["columns"]["corrected_y"] or
+                config["dataframe"]["columns"]["y"] (whichever is present).
             new_x_column (str, optional): Label of the destination 'X' column after
-                momentum calibration. Defaults to config["momentum"]["kx_column"].
+                momentum calibration. Defaults to config["dataframe"]["columns"]["kx"].
             new_y_column (str, optional): Label of the destination 'Y' column after
-                momentum calibration. Defaults to config["momentum"]["ky_column"].
+                momentum calibration. Defaults to config["dataframe"]["columns"]["ky"].
             calibration (dict, optional): Dictionary containing calibration parameters.
                 Defaults to 'self.calibration' or config["momentum"]["calibration"].
             suppress_output (bool, optional): Option to suppress output of diagnostic information.

sed/config/config_model.py

@@ -6,6 +6,7 @@
 from typing import Union
 
 from pydantic import BaseModel
+from pydantic import ConfigDict
 from pydantic import DirectoryPath
 from pydantic import Field
 from pydantic import field_validator
@@ -101,6 +102,8 @@ class DataframeModel(BaseModel):
     sector_id_reserved_bits: Optional[int] = None
     sector_delays: Optional[Sequence[int]] = None
 
+    # write validator for model so that x_column gets converted to columns: x
+
 
 class BinningModel(BaseModel):
     hist_mode: str
@@ -113,7 +116,7 @@ class BinningModel(BaseModel):
 class HistogramModel(BaseModel):
     bins: Sequence[int]
     axes: Sequence[str]
-    ranges: Sequence[Sequence[int]]
+    ranges: Sequence[tuple[float, float]]
 
 
 class StaticModel(BaseModel):
@@ -250,3 +253,5 @@ class ConfigModel(BaseModel):
     metadata: Optional[MetadataModel] = None
     nexus: Optional[NexusModel] = None
     static: Optional[StaticModel] = None
+
+    model_config = ConfigDict(extra="forbid")

sed/config/flash_example_config.yaml

@@ -1,13 +1,4 @@
 # This file contains the default configuration for the flash loader.
-
-# The paths to the raw and parquet data directories. If these are not
-# provided, the loader will try to find the data based on year beamtimeID etc
-paths:
-  # location of the raw data.
-  raw: ""
-  # location of the intermediate parquet files.
-  processed: ""
-
 core:
   # defines the loader
   loader: 'flash'
@@ -21,6 +12,13 @@ core:
   year: 2023
   # the instrument used
   instrument: hextof # hextof, wespe, etc
+  # The paths to the raw and parquet data directories. If these are not
+  # provided, the loader will try to find the data based on year beamtimeID etc
+  # paths:
+  # # location of the raw data.
+  #  raw: ""
+  # # location of the intermediate parquet files.
+  #  processed: ""
 
 binning:
   # Histogram computation mode to use.

sed/config/mpes_example_config.yaml

@@ -6,9 +6,9 @@ core:
   # Option to use the copy tool to mirror data to a local storage location before processing.
   use_copy_tool: False
   # path to the root of the source data directory
-  copy_tool_source: "/path/to/data/"
+  copy_tool_source: null # "/path/to/data/"
   # path to the root or the local data storage
-  copy_tool_dest: "/path/to/localDataStore/"
+  copy_tool_dest: null # "/path/to/localDataStore/"
   # optional keywords for the copy tool:
   copy_tool_kwds:
     # group id to set for copied files and folders

sed/core/config.py

@@ -30,8 +30,8 @@ def parse_config(
     system_config: dict | str = None,
     default_config: (dict | str) = f"{package_dir}/config/default.yaml",
     verbose: bool = True,
-    model: bool = False,
-) -> dict | ConfigModel:
+    verify_config: bool = True,
+) -> dict:
     """Load the config dictionary from a file, or pass the provided config dictionary.
     The content of the loaded config dictionary is then completed from a set of pre-configured
     config files in hierarchical order, by adding missing items. These additional config files
@@ -57,13 +57,13 @@ def parse_config(
             or file path. The loaded dictionary is completed with the default values.
             Defaults to *package_dir*/config/default.yaml".
         verbose (bool, optional): Option to report loaded config files. Defaults to True.
-        model (bool, optional): Option to return the config model instead of the dictionary.
+        verify_config (bool, optional): Option to verify config file. Defaults to True.
     Raises:
         TypeError: Raised if the provided file is neither *json* nor *yaml*.
         FileNotFoundError: Raised if the provided file is not found.
 
     Returns:
-        dict: Loaded and possibly completed config dictionary.
+        ConfigModel: Loaded and possibly completed pydantic config model.
     """
     if config is None:
         config = {}
@@ -144,10 +144,10 @@ def parse_config(
         base_dictionary=default_dict,
     )
 
+    if not verify_config:
+        return config_dict
     # Run the config through the ConfigModel to ensure it is valid
     config_model = ConfigModel(**config_dict)
-    if model:
-        return config_model
     return config_model.model_dump()
 
 

sed/dataset/dataset.py

@@ -55,6 +55,7 @@ def load_datasets_dict() -> dict:
             system_config={},
             default_config=DatasetsManager.json_path["module"],
             verbose=False,
+            verify_config=False,
         )
 
     @staticmethod

tests/data/loader/flash/config.yaml

@@ -61,7 +61,21 @@ dataframe:
   sector_delays: [0., 0., 0., 0., 0., 0., 0., 0.]
 
   jitter_cols: ["dldPosX", "dldPosY", "dldTimeSteps"]
-
+  columns:
+    x: dldPosX
+    corrected_x: X
+    kx: kx
+    y: dldPosY
+    corrected_y: Y
+    ky: ky
+    tof: dldTimeSteps
+    tof_ns: dldTime
+    corrected_tof: tm
+    timestamp: timeStamp
+    auxiliary: dldAux
+    sector_id: dldSectorID
+    delay: delayStage
+    corrected_delay: pumpProbeTime
   units:
     dldPosX: 'step'
     dldPosY: 'step'