Pydantic model

.cspell/custom-dictionary.txt

@@ -126,6 +126,7 @@ ftype
 fwhm
 genindex
 getgid
+getgrgid
 getmtime
 gpfs
 griddata

sed/config/config_model.py

@@ -1,4 +1,5 @@
 """Pydantic model to validate the config for SED package."""
+import grp
 from collections.abc import Sequence
 from typing import Literal
 from typing import Optional
@@ -18,29 +19,45 @@
 ## https://github.com/astral-sh/ruff/issues/5434
 
 
-class Paths(BaseModel):
+class PathsModel(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     raw: DirectoryPath
     processed: Optional[Union[DirectoryPath, NewPath]] = None
 
 
+class CopyToolModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    source: DirectoryPath
+    dest: DirectoryPath
+    safety_margin: Optional[float] = None
+    gid: Optional[int] = None
+    scheduler: Optional[str] = None
+
+    @field_validator("gid")
+    @classmethod
+    def validate_gid(cls, v: int) -> int:
+        """Checks if the gid is valid on the system"""
+        try:
+            grp.getgrgid(v)
+        except KeyError:
+            raise ValueError(f"Invalid value {v} for gid. Group not found.")
+        return v
+
+
 class CoreModel(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     loader: str
     verbose: Optional[bool] = None
-    paths: Optional[Paths] = None
+    paths: Optional[PathsModel] = None
     num_cores: Optional[int] = None
     year: Optional[int] = None
     beamtime_id: Optional[Union[int, str]] = None
     instrument: Optional[str] = None
     beamline: Optional[str] = None
-    # TODO: move copy tool to a separate model
-    use_copy_tool: Optional[bool] = None
-    copy_tool_source: Optional[DirectoryPath] = None
-    copy_tool_dest: Optional[DirectoryPath] = None
-    copy_tool_kwds: Optional[dict] = None
+    copy_tool: Optional[CopyToolModel] = None
 
     @field_validator("loader")
     @classmethod
@@ -51,6 +68,14 @@ def validate_loader(cls, v: str) -> str:
             raise ValueError(f"Invalid loader {v}. Available loaders are: {names}")
         return v
 
+    @field_validator("num_cores")
+    @classmethod
+    def validate_num_cores(cls, v: int) -> int:
+        """Checks if the num_cores field is a positive integer"""
+        if v < 1:
+            raise ValueError(f"Invalid value {v} for num_cores. Needs to be > 0.")
+        return v
+
 
 class ColumnsModel(BaseModel):
     model_config = ConfigDict(extra="forbid")

sed/config/mpes_example_config.yaml

@@ -4,15 +4,13 @@ core:
   # Number of parallel threads to use for parallelized jobs (e.g. binning, data conversion, copy, ...)
   num_cores: 20
   # Option to use the copy tool to mirror data to a local storage location before processing.
-  use_copy_tool: False
-  # path to the root of the source data directory
-  copy_tool_source: null # "/path/to/data/"
-  # path to the root or the local data storage
-  copy_tool_dest: null # "/path/to/localDataStore/"
-  # optional keywords for the copy tool:
-  copy_tool_kwds:
-    # group id to set for copied files and folders
-    gid: 1001
+  # copy_tool:
+    # # path to the root of the source data directory
+    # source: "/path/to/data/"
+    # # path to the root or the local data storage
+    # dest: "/path/to/localDataStore/"
+    # # group id to set for copied files and folders
+    # gid: 1000
 
 dataframe:
   # hdf5 group name containing eventIDs occurring at every millisecond (used to calculate timestamps)

sed/core/processor.py

@@ -161,22 +161,17 @@ def __init__(
             verbose=self._verbose,
         )
 
-        self.use_copy_tool = self._config.get("core", {}).get(
-            "use_copy_tool",
-            False,
-        )
+        self.use_copy_tool = "copy_tool" in self._config["core"]
         if self.use_copy_tool:
             try:
                 self.ct = CopyTool(
-                    source=self._config["core"]["copy_tool_source"],
-                    dest=self._config["core"]["copy_tool_dest"],
                     num_cores=self._config["core"]["num_cores"],
-                    **self._config["core"].get("copy_tool_kwds", {}),
+                    **self._config["core"]["copy_tool"],
                 )
                 logger.debug(
-                    f"Initialized copy tool: Copy file from "
-                    f"'{self._config['core']['copy_tool_source']}' "
-                    f"to '{self._config['core']['copy_tool_dest']}'.",
+                    f"Initialized copy tool: Copy files from "
+                    f"'{self._config['core']['copy_tool']['source']}' "
+                    f"to '{self._config['core']['copy_tool']['dest']}'.",
                 )
             except KeyError:
                 self.use_copy_tool = False

sed/loader/mpes/loader.py

@@ -72,7 +72,6 @@ def hdf5_to_dataframe(
 
     electron_channels = []
     column_names = []
-    print("Print values: ", channels)
     for name, channel in channels.items():
         if channel["format"] == "per_electron":
             if channel["dataset_key"] in test_proc:

tests/test_processor.py

@@ -219,7 +219,7 @@ def test_attributes_setters() -> None:
 
 def test_copy_tool() -> None:
     """Test the copy tool functionality in the processor"""
-    config = {"core": {"loader": "mpes", "use_copy_tool": True}}
+    config: dict[str, dict[str, Any]] = {"core": {"loader": "mpes"}}
     processor = SedProcessor(
         config=config,
         folder_config={},
@@ -231,10 +231,7 @@ def test_copy_tool() -> None:
     config = {
         "core": {
             "loader": "mpes",
-            "use_copy_tool": True,
-            "copy_tool_source": source_folder,
-            "copy_tool_dest": dest_folder,
-            "copy_tool_kwds": {"gid": os.getgid()},
+            "copy_tool": {"source": source_folder, "dest": dest_folder, "gid": os.getgid()},
         },
     }
     processor = SedProcessor(
@@ -248,17 +245,6 @@ def test_copy_tool() -> None:
     processor.load(files=files)
     assert processor.files[0].find(dest_folder) > -1
 
-    # test illegal keywords:
-    config["core"]["copy_tool_kwds"] = {"gid": os.getgid(), "illegal_keyword": True}
-    with pytest.raises(TypeError):
-        processor = SedProcessor(
-            config=config,
-            folder_config={},
-            user_config={},
-            system_config={},
-            verbose=True,
-        )
-
 
 feature4 = np.array([[203.2, 341.96], [299.16, 345.32], [304.38, 149.88], [199.52, 152.48]])
 feature5 = np.array(