Merge branch 'v1_feature_branch' into energy_calibration_bias_shift

.cspell/custom-dictionary.txt

@@ -37,6 +37,7 @@ bysource
 caldir
 calib
 calibdict
+caplog
 capsys
 cdeform
 cdeformfield

pyproject.toml

@@ -3,7 +3,7 @@ name = "sed-processor"
 packages = [
     {include = "sed"}
 ]
-version = "0.1.10a6"
+version = "0.1.10a8"
 description = "Single Event Data Frame Processor: Backend to handle photoelectron resolved datastreams"
 authors = ["OpenCOMPES team <[email protected]>"]
 readme = "README.md"
@@ -27,18 +27,17 @@ numba = ">=0.55.1"
 numpy = ">=1.18, <2.0"
 pandas = ">=1.4.1"
 psutil = ">=5.9.0"
-pynxtools-mpes = ">=0.0.3"
-pynxtools = ">=0.3.1"
+pynxtools-mpes = ">=0.1.1"
+pynxtools = ">=0.5.0"
 pyyaml = ">=6.0.0"
 scipy = ">=1.8.0"
-setuptools = ">70"
 symmetrize = ">=0.5.5"
 threadpoolctl = ">=3.1.0"
 tifffile = ">=2022.2.9"
 tqdm = ">=4.62.3"
 xarray = ">=0.20.2"
 joblib = ">=1.2.0"
-pyarrow = ">=14.0.1"
+pyarrow = ">=14.0.1, <17.0"
 jupyter = {version = ">=1.0.0", optional = true}
 ipykernel = {version = ">=6.9.1", optional = true}
 jupyterlab = {version = "^3.4.0", optional = true}

sed/config/NXmpes_config.json

@@ -2,7 +2,6 @@
   "/@default": "entry",
   "/ENTRY[entry]/@default": "data",
   "/ENTRY[entry]/definition": "NXmpes",
-  "/ENTRY[entry]/definition/@version": "None",
   "/ENTRY[entry]/title": "@attrs:metadata/entry_title",
   "/ENTRY[entry]/start_time": "@attrs:metadata/timing/acquisition_start",
   "/ENTRY[entry]/experiment_institution": "Fritz Haber Institute - Max Planck Society",
@@ -15,9 +14,9 @@
   "/ENTRY[entry]/collection_time": "@attrs:metadata/timing/collection_time",
   "/ENTRY[entry]/collection_time/@units": "s",
   "/ENTRY[entry]/USER[user]": {
-    "name": "@attrs:metadata/user0/name",
+    "name": "!@attrs:metadata/user0/name",
     "role": "@attrs:metadata/user0/role",
-    "affiliation": "@attrs:metadata/user0/affiliation",
+    "affiliation": "!@attrs:metadata/user0/affiliation",
     "address": "@attrs:metadata/user0/address",
     "email": "@attrs:metadata/user0/email"
   },
@@ -45,7 +44,7 @@
     "pressure_gauge": {
       "name": "sample_chamber_pressure",
       "measurement": "pressure",
-      "value": "@attrs:metadata/file/trARPES:XGS600:PressureAC:P_RD",
+      "value": "!@attrs:metadata/file/trARPES:XGS600:PressureAC:P_RD",
       "value/@units": "mbar"
     }
   },
@@ -63,19 +62,19 @@
     ],
     "slow_axes": "@attrs:metadata/instrument/analyzer/slow_axes",
     "energy_resolution": {
-      "resolution": "@attrs:metadata/instrument/analyzer/energy_resolution",
+      "resolution": "!@attrs:metadata/instrument/analyzer/energy_resolution",
       "resolution/@units": "meV",
       "physical_quantity": "energy",
       "type": "estimated"
     },
     "momentum_resolution": {
-      "resolution": "@attrs:metadata/instrument/analyzer/momentum_resolution",
+      "resolution": "!@attrs:metadata/instrument/analyzer/momentum_resolution",
       "resolution/@units": "1/angstrom",
       "physical_quantity": "momentum",
       "type": "estimated"
     },
     "spatial_resolution": {
-      "resolution": "@attrs:metadata/instrument/analyzer/spatial_resolution",
+      "resolution": "!@attrs:metadata/instrument/analyzer/spatial_resolution",
       "resolution/@units": "µm",
       "physical_quantity": "length",
       "type": "estimated"
@@ -173,7 +172,7 @@
   "/ENTRY[entry]/INSTRUMENT[instrument]/beamTYPE[beam_probe]": {
     "distance": 0.0,
     "distance/@units": "mm",
-    "incident_energy": "@attrs:metadata/instrument/beam/probe/incident_energy",
+    "incident_energy": "!@attrs:metadata/instrument/beam/probe/incident_energy",
     "incident_energy/@units": "eV",
     "incident_energy_spread": "@attrs:metadata/instrument/beam/probe/incident_energy_spread",
     "incident_energy_spread/@units": "eV",
@@ -197,7 +196,7 @@
   "/ENTRY[entry]/INSTRUMENT[instrument]/beamTYPE[beam_pump]": {
     "distance": 0.0,
     "distance/@units": "mm",
-    "incident_energy": "@attrs:metadata/instrument/beam/pump/incident_energy",
+    "incident_energy": "!@attrs:metadata/instrument/beam/pump/incident_energy",
     "incident_energy/@units": "eV",
     "incident_energy_spread": "@attrs:metadata/instrument/beam/pump/incident_energy_spread",
     "incident_energy_spread/@units": "eV",
@@ -221,13 +220,13 @@
     "temperature_sensor": {
       "name": "sample_temperature",
       "measurement": "temperature",
-      "value": "@attrs:metadata/file/trARPES:Carving:TEMP_RBV",
+      "value": "!@attrs:metadata/file/trARPES:Carving:TEMP_RBV",
       "value/@units": "K"
     },
     "sample_bias_voltmeter": {
       "name": "sample_bias",
       "measurement": "voltage",
-      "value": "@attrs:metadata/file/KTOF:Lens:Sample:V",
+      "value": "!@attrs:metadata/file/KTOF:Lens:Sample:V",
       "value/@units": "V"
     },
     "depends_on": "/entry/instrument/manipulator/transformations/trans_z",

sed/config/flash_example_config.yaml

@@ -8,7 +8,7 @@ core:
   # the beamline where experiment took place
   beamline: pg2
   # the ID number of the beamtime
-  beamtime_id: 11013410
+  beamtime_id: 11019101
   # the year of the beamtime
   year: 2023
   # the instrument used
@@ -18,9 +18,9 @@ core:
   # provided, the loader will try to find the data based on year beamtimeID etc
   paths:
     # location of the raw data.
-    data_raw_dir: "tests/data/loader/flash/"
+    data_raw_dir: ""
     # location of the intermediate parquet files.
-    data_parquet_dir: "tests/data/loader/flash/parquet"
+    data_parquet_dir: ""
 
 binning:
   # Histogram computation mode to use.

tests/test_processor.py

@@ -5,6 +5,7 @@
 import csv
 import glob
 import itertools
+import logging
 import os
 import tempfile
 from importlib.util import find_spec
@@ -15,6 +16,8 @@
 import numpy as np
 import pytest
 import xarray as xr
+import yaml
+from pynxtools.dataconverter.convert import ValidationFailed
 
 from sed import SedProcessor
 from sed.core.config import parse_config
@@ -1017,14 +1020,19 @@ def test_get_normalization_histogram() -> None:
 metadata["sample"]["name"] = "Sample Name"
 
 
-def test_save() -> None:
+def test_save(caplog) -> None:
     """Test the save functionality"""
     config = parse_config(
         config={"dataframe": {"tof_binning": 1}},
         folder_config={},
         user_config=package_dir + "/../sed/config/mpes_example_config.yaml",
         system_config={},
     )
+    config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"][
+        "MCPfront"
+    ] = 21.0
+    config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["Z1"] = 2450
+    config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["F"] = 69.23
     processor = SedProcessor(
         folder=df_folder,
         config=config,
@@ -1052,9 +1060,41 @@ def test_save() -> None:
     processor.save("output.h5")
     assert os.path.isfile("output.h5")
     os.remove("output.h5")
+    # convert using the fail=True keyword. This ensures that the pynxtools backend will throw
+    # and error if any validation problems occur.
     processor.save(
         "output.nxs",
         input_files=df_folder + "../../../../sed/config/NXmpes_config.json",
+        fail=True,
     )
     assert os.path.isfile("output.nxs")
+    # Test that a validation error is raised if a required field is missing
+    del processor.binned.attrs["metadata"]["sample"]["name"]
+    with pytest.raises(ValidationFailed):
+        processor.save(
+            "result.nxs",
+            input_files=df_folder + "../../../../sed/config/NXmpes_config.json",
+            fail=True,
+        )
+    # Check that the issues are raised as warnings per default:
+    caplog.clear()
+    with open("temp_eln.yaml", "w") as f:
+        yaml.dump({"Instrument": {"undocumented_field": "undocumented entry"}}, f)
+    with caplog.at_level(logging.WARNING):
+        processor.save(
+            "result.nxs",
+            input_files=[df_folder + "../../../../sed/config/NXmpes_config.json"],
+            eln_data="temp_eln.yaml",
+        )
+        assert (
+            caplog.messages[0]
+            == "The data entry corresponding to /ENTRY[entry]/SAMPLE[sample]/name is required and "
+            "hasn't been supplied by the reader."
+        )
+        assert (
+            caplog.messages[1]
+            == "Field /ENTRY[entry]/INSTRUMENT[instrument]/undocumented_field written without "
+            "documentation."
+        )
     os.remove("output.nxs")
+    os.remove("temp_eln.yaml")

tutorial/4_hextof_workflow.ipynb

@@ -47,7 +47,16 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Download and unpack data"
+    "### Get data paths"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The paths are such that if you are on Maxwell, it uses those. Otherwise data is downloaded in current directory from Zenodo.\n",
+    "\n",
+    "Generally, if it is your beamtime, you can both read the raw data and write to processed directory. However, for the public data, you can not write to processed directory."
    ]
   },
   {
@@ -56,10 +65,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# data_path can be defined and used to store the data in a specific location\n",
-    "dataset.get(\"Gd_W110\") # Put in Path to a storage of at least 10 GByte free space.\n",
-    "path = dataset.dir\n",
-    "print(path)"
+    "beamtime_dir = \"/asap3/flash/gpfs/pg2/2023/data/11019101\" # on Maxwell\n",
+    "if os.path.exists(beamtime_dir) and os.access(beamtime_dir, os.R_OK):\n",
+    "    path = beamtime_dir + \"/raw/hdf/offline/fl1user3\"\n",
+    "    buffer_path = \"Gd_W110/processed/\"\n",
+    "else:\n",
+    "    # data_path can be defined and used to store the data in a specific location\n",
+    "    dataset.get(\"Gd_W110\") # Put in Path to a storage of at least 10 GByte free space.\n",
+    "    path = dataset.dir\n",
+    "    buffer_path = path + \"/processed/\""
    ]
   },
   {
@@ -99,35 +113,13 @@
     "config_override = {\n",
     "    \"core\": {\n",
     "        \"paths\": {\n",
-    "            \"data_raw_dir\": \"\",\n",
-    "            \"data_parquet_dir\": \"\"\n",
+    "            \"data_raw_dir\": path,\n",
+    "            \"data_parquet_dir\": buffer_path,\n",
     "        },\n",
     "    },\n",
     "}"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "If it is your beamtime, you can access both read the raw data and write to processed directory. For the public data, you can not write to processed directory."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "config_override['core']['paths']['data_raw_dir'] = \"/asap3/flash/gpfs/pg2/2023/data/11019101/raw/hdf/offline/fl1user3\"\n",
-    "# If this will not work for you, please change it to a path where you have write access\n",
-    "config_override['core']['paths']['data_parquet_dir'] = \"/asap3/flash/gpfs/pg2/2023/data/11019101/processed\"\n",
-    "# So we write to user space\n",
-    "config_override['core']['paths']['data_parquet_dir'] = path + \"/processed\"\n",
-    "# If you aren't using maxwell and downloaded the data, use this path\n",
-    "config_override['core']['paths']['data_raw_dir'] = path"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -766,7 +758,7 @@
     "    constant=-1463.7, # this is time zero\n",
     "    flip_delay_axis=True, # invert the direction of the delay axis\n",
     "    columns=['bam'], # use the bam to offset the values\n",
-    "    weights=[0.001], # bam is in fs, delay in ps\n",
+    "    weights=[-0.001], # bam is in fs, delay in ps\n",
     "    preserve_mean=True # preserve the mean of the delay axis\n",
     ")"
    ]
@@ -967,7 +959,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "python3",
+   "display_name": "Python 3.9",
    "language": "python",
    "name": "python3"
   },
@@ -985,5 +977,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }