Update documentation

sed/develop/_sources/sed/config.rst.txt

@@ -1,6 +1,6 @@
 Config
 ***************************************************
-.. automodule:: sed.core.config
+.. automobile:: sed.core.config
    :members:
    :undoc-members:
 

sed/develop/_sources/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb.txt

@@ -59,7 +59,7 @@
    "outputs": [],
    "source": [
     "# create sed processor using the config file:\n",
-    "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", verbose=True)"
+    "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={}, verbose=True)"
    ]
   },
   {
@@ -466,9 +466,9 @@
    "metadata": {},
    "source": [
     "#### 3. Step:\n",
-    "Next, the detected peak positions and bias voltages are used to determine the calibration function. This can be either done by fitting the functional form d^2/(t-t0)^2 via lmfit (\"lmfit\"), or using a polynomial approximation (\"lstsq\" or \"lsqr\"). Here, one can also define a reference id, and a reference energy. Those define the absolute energy position of the feature used for calibration in the \"reference\" trace, at the bias voltage where the final measurement has been performed. The energy scale can be either \"kinetic\" (decreasing energy with increasing TOF), or \"binding\" (increasing energy with increasing TOF).\n",
+    "Next, the detected peak positions and bias voltages are used to determine the calibration function. Essentially, the functional Energy(TOF) is being determined by either least-squares fitting of the functional form d^2/(t-t0)^2 via lmfit (``method``: \"lmfit\"), or by analytically obtaining a polynomial approximation (``method``: \"lstsq\" or \"lsqr\"). The parameter ``ref_energy`` is used to define the absolute energy position of the feature used for calibration in the calibrated energy scale. ``energy_scale`` can be either \"kinetic\" (decreasing energy with increasing TOF), or \"binding\" (increasing energy with increasing TOF).\n",
     "\n",
-    "After calculating the calibration, all traces corrected with the calibration are plotted ontop of each other, the calibration function together with the extracted features is plotted."
+    "After calculating the calibration, all traces corrected with the calibration are plotted ontop of each other, and the calibration function (Energy(TOF)) together with the extracted features is being plotted."
    ]
   },
   {
@@ -478,21 +478,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# use the refid of the bias that the measurement was taken at\n",
-    "# Eref can be used to set the absolute energy (kinetic energy, E-EF) of the feature used for energy calibration (if known)\n",
-    "refid=4\n",
-    "Eref=-0.5\n",
+    "# Eref can be used to set the absolute energy (kinetic energy, E-EF, etc.) of the feature used for energy calibration (if known)\n",
+    "Eref=-1.3\n",
     "# the lmfit method uses a fit of (d/(t-t0))**2 to determine the energy calibration\n",
     "# limits and starting values for the fitting parameters can be provided as dictionaries\n",
     "sp.calibrate_energy_axis(\n",
-    "    ref_id=refid,\n",
     "    ref_energy=Eref,\n",
     "    method=\"lmfit\",\n",
     "    energy_scale='kinetic',\n",
     "    d={'value':1.0,'min': .7, 'max':1.2, 'vary':True},\n",
     "    t0={'value':8e-7, 'min': 1e-7, 'max': 1e-6, 'vary':True},\n",
     "    E0={'value': 0., 'min': -100, 'max': 0, 'vary': True},\n",
-    "    verbose=True,\n",
     ")"
    ]
   },
@@ -523,7 +519,7 @@
    "metadata": {},
    "source": [
     "#### 4. Step:\n",
-    "Finally, the the energy axis is added to the dataframe."
+    "Finally, the the energy axis is added to the dataframe. Here, the applied bias voltages of the measurement is taken into account to provide the correct energy offset. If the bias cannot be read from the file, it can be provided manually."
    ]
   },
   {
@@ -533,7 +529,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sp.append_energy_axis()"
+    "sp.append_energy_axis(bias_voltage=16.8)"
    ]
   },
   {
@@ -546,6 +542,16 @@
     "The delay axis is calculated from the ADC input column based on the provided delay range. ALternatively, the delay scan range can also be extracted from attributes inside a source file, if present."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1619cbc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sp.dataframe.head()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -651,9 +657,6 @@
   }
  ],
  "metadata": {
-  "interpreter": {
-   "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243"
-  },
   "kernelspec": {
    "display_name": "python3",
    "language": "python",
@@ -669,7 +672,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.12"
+   "version": "3.9.19"
   }
  },
  "nbformat": 4,

sed/develop/_sources/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb.txt

@@ -143,7 +143,7 @@
    "outputs": [],
    "source": [
     "# create sed processor using the config file, and collect the meta data from the files:\n",
-    "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", metadata=metadata, collect_metadata=True)"
+    "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={}, metadata=metadata, collect_metadata=True)"
    ]
   },
   {
@@ -220,7 +220,7 @@
    "outputs": [],
    "source": [
     "# Apply stored config energy calibration\n",
-    "sp.append_energy_axis()"
+    "sp.append_energy_axis(bias_voltage=16.8)"
    ]
   },
   {
@@ -290,9 +290,6 @@
   }
  ],
  "metadata": {
-  "interpreter": {
-   "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243"
-  },
   "kernelspec": {
    "display_name": "python3",
    "language": "python",
@@ -308,7 +305,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.12"
+   "version": "3.9.19"
   }
  },
  "nbformat": 4,

sed/develop/_sources/tutorial/4_hextof_workflow.ipynb.txt

@@ -31,6 +31,8 @@
    },
    "outputs": [],
    "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
     "from typing import List\n",
     "from pathlib import Path\n",
     "import os\n",
@@ -68,11 +70,13 @@
     "beamtime_dir = \"/asap3/flash/gpfs/pg2/2023/data/11019101\" # on Maxwell\n",
     "if os.path.exists(beamtime_dir) and os.access(beamtime_dir, os.R_OK):\n",
     "    path = beamtime_dir + \"/raw/hdf/offline/fl1user3\"\n",
+    "    meta_path = beamtime_dir + \"/shared\"\n",
     "    buffer_path = \"Gd_W110/processed/\"\n",
     "else:\n",
     "    # data_path can be defined and used to store the data in a specific location\n",
     "    dataset.get(\"Gd_W110\") # Put in Path to a storage of at least 10 GByte free space.\n",
     "    path = dataset.dir\n",
+    "    meta_path = path\n",
     "    buffer_path = path + \"/processed/\""
    ]
   },
@@ -113,8 +117,8 @@
     "config_override = {\n",
     "    \"core\": {\n",
     "        \"paths\": {\n",
-    "            \"data_raw_dir\": path,\n",
-    "            \"data_parquet_dir\": buffer_path,\n",
+    "            \"raw\": path,\n",
+    "            \"processed\": buffer_path,\n",
     "        },\n",
     "    },\n",
     "}"
@@ -167,7 +171,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sp = SedProcessor(runs=[44762], config=config_override, system_config=config_file, collect_metadata=False, force_recreate=True)\n",
+    "sp = SedProcessor(runs=[44762], config=config_override, system_config=config_file, collect_metadata=False)\n",
     "# You can set collect_metadata=True if the scicat_url and scicat_token are defined"
    ]
   },
@@ -497,14 +501,12 @@
    "outputs": [],
    "source": [
     "sp.calibrate_energy_axis(\n",
-    "    ref_id=4,\n",
     "    ref_energy=-.55,\n",
     "    method=\"lmfit\",\n",
     "    energy_scale='kinetic',\n",
     "    d={'value':1.0,'min': .2, 'max':1.0, 'vary':False},\n",
     "    t0={'value':5e-7, 'min': 1e-7, 'max': 1e-6, 'vary':True},\n",
     "    E0={'value': 0., 'min': -100, 'max': 100, 'vary': True},\n",
-    "    verbose=True,\n",
     ")"
    ]
   },
@@ -577,7 +579,7 @@
    "metadata": {},
    "source": [
     "### correct offsets\n",
-    "The energy axis is now correct, but still the curves do not stack on each other as we are not compensating for the `sampleBias`. In the same way, we can compensate the photon energy (`monochromatorPhotonEnergy`) and the `tofVoltage` "
+    "The energy axis is now correct, taking the sample bias of the measurement into account. Additionally, we can compensate the photon energy (`monochromatorPhotonEnergy`) and the `tofVoltage`."
    ]
   },
   {
@@ -587,10 +589,9 @@
    "outputs": [],
    "source": [
     "sp.add_energy_offset(\n",
-    "    constant=-32, # Sample bias used as reference for energy calibration\n",
-    "    columns=['sampleBias','monochromatorPhotonEnergy','tofVoltage'],\n",
-    "    weights=[1,-1,-1],\n",
-    "    preserve_mean=[False, True, True],\n",
+    "    columns=['monochromatorPhotonEnergy','tofVoltage'],\n",
+    "    weights=[-1,-1],\n",
+    "    preserve_mean=[True, True],\n",
     ")"
    ]
   },
@@ -793,7 +794,7 @@
     "axes = ['energy','delayStage']\n",
     "bins = [100,150]\n",
     "delay_start,delay_stop=1462.00,1464.85\n",
-    "ranges = [[-3,2], [-1.15, 1.7]]\n",
+    "ranges = [[-3,2], [-1.1, 1.75]]\n",
     "res = sp.compute(bins=bins, axes=axes, ranges=ranges)"
    ]
   },
@@ -863,6 +864,26 @@
     "Once all the calibrations are done, a new run can be loaded by simply calling all the calibration functions."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sed.core.config import load_config\n",
+    "import numpy as np\n",
+    "metadata = load_config(meta_path + \"/44824_20230324T060430.json\")\n",
+    "\n",
+    "# Fix metadata\n",
+    "metadata[\"scientificMetadata\"][\"Laser\"][\"wavelength\"][\"value\"] = float(metadata[\"scientificMetadata\"][\"Laser\"][\"wavelength\"][\"value\"][:-2])\n",
+    "metadata[\"scientificMetadata\"][\"Laser\"][\"energy\"] = {\"value\": 1239.84/metadata[\"scientificMetadata\"][\"Laser\"][\"wavelength\"][\"value\"], \"unit\": \"eV\"}\n",
+    "metadata[\"scientificMetadata\"][\"Laser\"][\"polarization\"] = [1, 1, 0, 0]\n",
+    "metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_x\"] = float(metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_x\"])\n",
+    "metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_y\"] = float(metadata[\"scientificMetadata\"][\"Collection\"][\"field_aperture_y\"])\n",
+    "metadata[\"pi\"] = {\"institute\": \"JGU Mainz\"}\n",
+    "metadata[\"proposer\"] = {\"institute\": \"TU Dortmund\"}\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -877,7 +898,8 @@
     "    runs=[44824,44825,44826,44827],\n",
     "    config=config_override,\n",
     "    system_config=config_file,\n",
-    "    collect_metadata=False\n",
+    "    metadata = metadata,\n",
+    "    collect_metadata=False,\n",
     ")"
    ]
   },
@@ -915,7 +937,7 @@
     "axes = ['energy','delayStage']\n",
     "bins = [100,150]\n",
     "delay_start,delay_stop=1462.00,1464.85\n",
-    "ranges = [[-5,2], [-1.15, 1.7]]\n",
+    "ranges = [[-5,2], [-1.1, 1.75]]\n",
     "res = sp.compute(bins=bins, axes=axes, ranges=ranges, normalize_to_acquisition_time=\"delayStage\")"
    ]
   },
@@ -945,7 +967,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sp.save('binned.h5')"
+    "sp.save('runs44824-27.h5')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sp.save('runs44824-27.tiff')"
    ]
   },
   {
@@ -954,13 +985,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sp.save('binned.tiff')"
+    "sp.save(\"runs44824-27.nxs\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.9",
+   "display_name": "python3",
    "language": "python",
    "name": "python3"
   },

sed/develop/_sources/tutorial/5_sxp_workflow.ipynb.txt

@@ -99,8 +99,8 @@
     "config_override = {\n",
     "    \"core\": {\n",
     "        \"paths\": {\n",
-    "            \"data_raw_dir\": path,\n",
-    "            \"data_parquet_dir\": buffer_path,\n",
+    "            \"raw\": path,\n",
+    "            \"processed\": buffer_path,\n",
     "        },\n",
     "    },\n",
     "}"
@@ -403,7 +403,6 @@
    "outputs": [],
    "source": [
     "sp.calibrate_energy_axis(\n",
-    "    ref_id=5,\n",
     "    ref_energy=-33,\n",
     "    method=\"lmfit\",\n",
     "    energy_scale='kinetic',\n",
@@ -447,7 +446,14 @@
     "sp.load(runs=np.arange(58, 62))\n",
     "sp.add_jitter()\n",
     "sp.filter_column(\"pulseId\", max_value=756)\n",
-    "sp.append_energy_axis()"
+    "sp.append_energy_axis(bias_voltage=957)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we can bin as function fo energy and delay stage position"
    ]
   },
   {

sed/develop/_sources/tutorial/6_binning_with_time-stamped_data.ipynb.txt

@@ -68,7 +68,7 @@
    "outputs": [],
    "source": [
     "# create sed processor using the config file with time-stamps:\n",
-    "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", time_stamps=True)"
+    "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", system_config={}, time_stamps=True, verbose=True)"
    ]
   },
   {
@@ -157,12 +157,12 @@
    "source": [
     "# Load energy calibration EDCs\n",
     "scans = np.arange(127,136)\n",
-    "voltages = np.arange(22,13,-1)\n",
+    "voltages = np.arange(21,12,-1)\n",
     "files = [caldir + r'/Scan' + str(num).zfill(4) + '_1.h5' for num in scans]\n",
     "sp.load_bias_series(data_files=files, normalize=True, biases=voltages, ranges=[(64000, 76000)])\n",
     "rg = (65500, 66000)\n",
     "sp.find_bias_peaks(ranges=rg, ref_id=5, infer_others=True, apply=True)\n",
-    "sp.calibrate_energy_axis(ref_energy=-0.5, ref_id=4, energy_scale=\"kinetic\", method=\"lmfit\")"
+    "sp.calibrate_energy_axis(ref_energy=-0.5, energy_scale=\"kinetic\", method=\"lmfit\")"
    ]
   },
   {
@@ -173,6 +173,7 @@
    "outputs": [],
    "source": [
     "# Apply stored config energy calibration\n",
+    "#sp.append_energy_axis(bias_voltage=17)\n",
     "sp.append_energy_axis()"
    ]
   },