diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 5a840f3b..9bdcdec1 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -61,6 +61,9 @@ class CoreModel(BaseModel): instrument: Optional[str] = None beamline: Optional[str] = None copy_tool: Optional[CopyToolModel] = None + stream_name_prefixes: Optional[dict] = None + stream_name_postfixes: Optional[dict] = None + beamtime_dir: Optional[dict] = None @field_validator("loader") @classmethod @@ -159,17 +162,6 @@ class HistogramModel(BaseModel): ranges: Sequence[tuple[float, float]] -class StaticModel(BaseModel): - """Static configuration settings that shouldn't be changed by users.""" - - # flash specific settings - model_config = ConfigDict(extra="forbid") - - stream_name_prefixes: Optional[dict] = None - stream_name_postfixes: Optional[dict] = None - beamtime_dir: Optional[dict] = None - - class EnergyModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -360,4 +352,3 @@ class ConfigModel(BaseModel): histogram: HistogramModel metadata: Optional[MetadataModel] = None nexus: Optional[NexusModel] = None - static: Optional[StaticModel] = None diff --git a/sed/config/flash_example_config.yaml b/sed/config/flash_example_config.yaml index 0f12402b..823a53fe 100644 --- a/sed/config/flash_example_config.yaml +++ b/sed/config/flash_example_config.yaml @@ -19,6 +19,19 @@ core: # raw: "" # # location of the intermediate parquet files. # processed: "" + # The prefixes of the stream names for different DAQ systems for parsing filenames + stream_name_prefixes: + pbd: "GMD_DATA_gmd_data" + pbd2: "FL2PhotDiag_pbd2_gmd_data" + fl1user1: "FLASH1_USER1_stream_2" + fl1user2: "FLASH1_USER2_stream_2" + fl1user3: "FLASH1_USER3_stream_2" + fl2user1: "FLASH2_USER1_stream_2" + fl2user2: "FLASH2_USER2_stream_2" + # The beamtime directories for different DAQ systems. + # (Not to be changed by user) + beamtime_dir: + pg2: "/asap3/flash/gpfs/pg2/" binning: # Histogram computation mode to use. @@ -204,21 +217,4 @@ dataframe: # nexus: # reader: "mpes" # definition: "NXmpes" -# input_files: ["NXmpes_config_HEXTOF_light.json"] - -# (Not to be changed by user) -static: - # The prefixes of the stream names for different DAQ systems for parsing filenames - stream_name_prefixes: - pbd: "GMD_DATA_gmd_data" - pbd2: "FL2PhotDiag_pbd2_gmd_data" - fl1user1: "FLASH1_USER1_stream_2" - fl1user2: "FLASH1_USER2_stream_2" - fl1user3: "FLASH1_USER3_stream_2" - fl2user1: "FLASH2_USER1_stream_2" - fl2user2: "FLASH2_USER2_stream_2" - - # The beamtime directories for different DAQ systems. - # (Not to be changed by user) - beamtime_dir: - pg2: "/asap3/flash/gpfs/pg2/" +# input_files: ["NXmpes_config_HEXTOF_light.json"] \ No newline at end of file diff --git a/sed/config/sxp_example_config.yaml b/sed/config/sxp_example_config.yaml index b9e4d298..9b48c28c 100644 --- a/sed/config/sxp_example_config.yaml +++ b/sed/config/sxp_example_config.yaml @@ -6,6 +6,12 @@ core: year: 202302 beamline: sxp instrument: sxp + stream_name_prefixes: + DA03: "RAW-R" + stream_name_postfixes: + DA03: "-DA03-" + beamtime_dir: + sxp: "/gpfs/exfel/exp/SXP/" paths: raw: "/path/to/data" # change this to a local directory where you want to store the parquet files @@ -102,12 +108,3 @@ dataframe: format: per_train dataset_key: "/CONTROL/SCS_ILH_LAS/MDL/OPTICALDELAY_PP800/actualPosition/value" index_key: "/INDEX/trainId" - -static: - stream_name_prefixes: - DA03: "RAW-R" - stream_name_postfixes: - DA03: "-DA03-" - - beamtime_dir: - sxp: "/gpfs/exfel/exp/SXP/" diff --git a/sed/loader/flash/loader.py b/sed/loader/flash/loader.py index f451783f..6c84fc34 100644 --- a/sed/loader/flash/loader.py +++ b/sed/loader/flash/loader.py @@ -110,7 +110,7 @@ def _initialize_dirs(self) -> None: ) from exc beamtime_dir = Path( - self._config["static"]["beamtime_dir"][self._config["core"]["beamline"]], + self._config["core"]["beamtime_dir"][self._config["core"]["beamline"]], ) beamtime_dir = beamtime_dir.joinpath(f"{year}/data/{beamtime_id}/") @@ -175,7 +175,7 @@ def get_files_from_run_id( # type: ignore[override] FileNotFoundError: If no files are found for the given run in the directory. """ # Define the stream name prefixes based on the data acquisition identifier - stream_name_prefixes = self._config["static"]["stream_name_prefixes"] + stream_name_prefixes = self._config["core"]["stream_name_prefixes"] if folders is None: folders = self._config["core"]["base_folder"] diff --git a/sed/loader/sxp/loader.py b/sed/loader/sxp/loader.py index 83d7c97c..9ff0ac9a 100644 --- a/sed/loader/sxp/loader.py +++ b/sed/loader/sxp/loader.py @@ -116,7 +116,7 @@ def _initialize_dirs(self): ) from exc beamtime_dir = Path( - self._config["static"]["beamtime_dir"][self._config["core"]["beamline"]], + self._config["core"]["beamtime_dir"][self._config["core"]["beamline"]], ) beamtime_dir = beamtime_dir.joinpath(f"{year}/{beamtime_id}/") @@ -158,8 +158,8 @@ def get_files_from_run_id( FileNotFoundError: If no files are found for the given run in the directory. """ # Define the stream name prefixes based on the data acquisition identifier - stream_name_prefixes = self._config["static"]["stream_name_prefixes"] - stream_name_postfixes = self._config["static"].get("stream_name_postfixes", {}) + stream_name_prefixes = self._config["core"]["stream_name_prefixes"] + stream_name_postfixes = self._config["core"].get("stream_name_postfixes", {}) if isinstance(run_id, (int, np.integer)): run_id = str(run_id).zfill(4) diff --git a/tests/data/loader/flash/config.yaml b/tests/data/loader/flash/config.yaml index 8c57924b..fbbcba25 100644 --- a/tests/data/loader/flash/config.yaml +++ b/tests/data/loader/flash/config.yaml @@ -17,6 +17,21 @@ core: # beamtime_id: xxxxxxxx # year: 20xx + # The prefixes of the stream names for different DAQ systems for parsing filenames + stream_name_prefixes: + pbd: "GMD_DATA_gmd_data" + pbd2: "FL2PhotDiag_pbd2_gmd_data" + fl1user1: "FLASH1_USER1_stream_2" + fl1user2: "FLASH1_USER2_stream_2" + fl1user3: "FLASH1_USER3_stream_2" + fl2user1: "FLASH2_USER1_stream_2" + fl2user2: "FLASH2_USER2_stream_2" + + # The beamtime directories for different DAQ systems. + # (Not to be changed by user) + beamtime_dir: + pg2: "/asap3/flash/gpfs/pg2/" + dataframe: # The name of the DAQ system to use. Necessary to resolve the filenames/paths. @@ -141,21 +156,3 @@ dataframe: index_key: "/FL1/Photon Diagnostic/GMD/Pulse resolved energy/energy tunnel/index" dataset_key: "/FL1/Photon Diagnostic/GMD/Pulse resolved energy/energy tunnel/value" slice: 0 - - -# (Not to be changed by user) -static: - # The prefixes of the stream names for different DAQ systems for parsing filenames - stream_name_prefixes: - pbd: "GMD_DATA_gmd_data" - pbd2: "FL2PhotDiag_pbd2_gmd_data" - fl1user1: "FLASH1_USER1_stream_2" - fl1user2: "FLASH1_USER2_stream_2" - fl1user3: "FLASH1_USER3_stream_2" - fl2user1: "FLASH2_USER1_stream_2" - fl2user2: "FLASH2_USER2_stream_2" - - # The beamtime directories for different DAQ systems. - # (Not to be changed by user) - beamtime_dir: - pg2: "/asap3/flash/gpfs/pg2/" diff --git a/tests/data/loader/sxp/config.yaml b/tests/data/loader/sxp/config.yaml index c5680847..2f88bc50 100644 --- a/tests/data/loader/sxp/config.yaml +++ b/tests/data/loader/sxp/config.yaml @@ -5,6 +5,13 @@ core: raw: "tests/data/loader/sxp/" processed: "tests/data/loader/sxp/parquet" num_cores: 10 + stream_name_prefixes: + DA03: "RAW-R" + stream_name_postfixes: + DA03: "-DA03-" + + beamtime_dir: + sxp: "/GPFS/exfel/exp/SXP/" dataframe: ubid_offset: 0 @@ -82,12 +89,3 @@ dataframe: format: per_train dataset_key: "/CONTROL/SCS_ILH_LAS/MDL/OPTICALDELAY_PP800/actualPosition/value" index_key: "/INDEX/trainId" - -static: - stream_name_prefixes: - DA03: "RAW-R" - stream_name_postfixes: - DA03: "-DA03-" - - beamtime_dir: - sxp: "/GPFS/exfel/exp/SXP/" diff --git a/tests/loader/flash/test_flash_loader.py b/tests/loader/flash/test_flash_loader.py index 3d9a5170..de0bdf35 100644 --- a/tests/loader/flash/test_flash_loader.py +++ b/tests/loader/flash/test_flash_loader.py @@ -33,7 +33,7 @@ def test_initialize_dirs( config_["core"]["year"] = "2000" # Find base path of beamline from config. Here, we use pg2 - base_path = config_["static"]["beamtime_dir"]["pg2"] + base_path = config_["core"]["beamtime_dir"]["pg2"] expected_path = ( Path(base_path) / config_["core"]["year"] / "data" / config_["core"]["beamtime_id"] ) diff --git a/tests/loader/sxp/test_sxp_loader.py b/tests/loader/sxp/test_sxp_loader.py index 85a31eac..1ee06d41 100644 --- a/tests/loader/sxp/test_sxp_loader.py +++ b/tests/loader/sxp/test_sxp_loader.py @@ -87,7 +87,7 @@ def test_initialize_dirs(config_file: dict, fs) -> None: config["core"]["year"] = "2000" # Find base path of beamline from config. - base_path = config["static"]["beamtime_dir"]["sxp"] + base_path = config["core"]["beamtime_dir"]["sxp"] expected_path = Path(base_path) / config["core"]["year"] / config["core"]["beamtime_id"] # Create expected paths expected_raw_path = expected_path / "raw"