diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index 128efab7..76abda61 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -286,6 +286,7 @@ ptargs pullrequest pval pyarrow +pydata pyenv pygments pynxtools @@ -370,6 +371,7 @@ utime varnames venv verts +viewcode vmax voxels VTOF diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 6c417d67..8a7124ca 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -3,6 +3,7 @@ on: # Triggers the workflow on push but only for the main branch push: branches: [ main ] + tags: [ v* ] paths: - sed/**/* - tutorial/** @@ -11,18 +12,6 @@ on: workflow_dispatch: - # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - jobs: build: runs-on: ubuntu-latest @@ -47,8 +36,8 @@ jobs: - name: "Setup Python, Poetry and Dependencies" uses: packetcoders/action-setup-cache-python-poetry@main with: - python-version: 3.8 - poetry-version: 1.2.2 + python-version: 3.9 + poetry-version: 1.8.3 - name: Install notebook dependencies run: poetry install -E notebook --with docs @@ -65,13 +54,6 @@ jobs: cp -r $GITHUB_WORKSPACE/sed/config $GITHUB_WORKSPACE/docs/sed rm $GITHUB_WORKSPACE/docs/tutorial/5_sxp_workflow.ipynb - # To be included later - # - name: Cache docs build - # id: cache-docs - # uses: actions/cache@v3 - # with: - # path: $GITHUB_WORKSPACE/_build - # key: ${{ runner.os }}-docs - name: download RAW data # if: steps.cache-primes.outputs.cache-hit != 'true' @@ -84,25 +66,88 @@ jobs: cd $GITHUB_WORKSPACE/docs poetry run python scripts/build_flash_parquets.py + # to be removed later. This theme doesn't support <3.9 python and our lock file contains 3.8 + - name: install pydata-sphinx-theme + run: | + poetry run pip install pydata-sphinx-theme + + - name: Change version for develop build + if: startsWith(github.ref, 'refs/heads/') && github.ref != 'refs/heads/main' + run: | + VERSION=`sed -n 's/^version = "\(.*\)".*/\1/p' $GITHUB_WORKSPACE/pyproject.toml` + MOD_VERSION=$VERSION".dev0" + echo $MOD_VERSION + sed -i "s/^version = \"$VERSION\"/version = \"$MOD_VERSION\"/" $GITHUB_WORKSPACE/pyproject.toml + - name: build Sphinx docs run: poetry run sphinx-build -b html $GITHUB_WORKSPACE/docs $GITHUB_WORKSPACE/_build - - name: Setup Pages - uses: actions/configure-pages@v4 - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-artifact@v4 with: - path: '_build' + name: sphinx-docs + path: _build - # Deployment job - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + # this job pushes the built documentation to the docs repository + push: runs-on: ubuntu-latest needs: build steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 + - name: Checkout docs repo + uses: actions/checkout@v2 + with: + repository: ${{ github.repository_owner }}/docs + token: ${{ secrets.GITHUB_TOKEN }} + path: 'docs-repo' + + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Setup SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_DOCS_DEPLOY_KEY }} + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: sphinx-docs + path: sphinx-docs + + - name: Determine version folder + id: version-folder + run: | + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + echo "folder=sed/$VERSION" >> $GITHUB_OUTPUT + rm docs-repo/sed/stable + ln -s -r docs-repo/sed/$VERSION docs-repo/sed/stable + elif [[ $GITHUB_REF == refs/heads/main ]]; then + echo "folder=sed/latest" >> $GITHUB_OUTPUT + else + echo "folder=sed/develop" >> $GITHUB_OUTPUT + fi + + - name: Update switcher.json + run: | + VERSION=`grep "SED documentation." sphinx-docs/index.html | sed -n 's/.*SED \(.*\) documentation.*/\1/p'` + echo "python docs-repo/sed/update_switcher.py docs-repo/sed/switcher.json $GITHUB_REF $VERSION" + python docs-repo/sed/update_switcher.py docs-repo/sed/switcher.json $GITHUB_REF $VERSION + + - name: Copy documentation to the right version folder + run: | + mkdir -p docs-repo/${{ steps.version-folder.outputs.folder }} + cp -r sphinx-docs/* docs-repo/${{ steps.version-folder.outputs.folder }} + rm -rf docs-repo/${{ steps.version-folder.outputs.folder }}/.doctrees + rm -rf docs-repo/${{ steps.version-folder.outputs.folder }}/tutorial/*.ipynb + + - name: Push changes + run: | + cd docs-repo + git config user.name github-actions + git config user.email github-actions@github.com + git add . + git commit -m "Update documentation" + git push diff --git a/.gitignore b/.gitignore index 59fa215f..e7828d0f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ *.hdf5 *.nxs *.nx -*.nxs +*.parquet *.zip **/datasets/* **/processed/* diff --git a/README.md b/README.md index 848a5405..1ff96bb7 100644 --- a/README.md +++ b/README.md @@ -7,23 +7,21 @@ [![](https://img.shields.io/pypi/v/sed-processor)](https://pypi.org/project/sed-processor) [![Coverage Status](https://coveralls.io/repos/github/OpenCOMPES/sed/badge.svg?branch=main&kill_cache=1)](https://coveralls.io/github/OpenCOMPES/sed?branch=main) -Backend to handle photoelectron resolved datastreams. +**sed-processor** is a backend to process and bin multidimensional single-event datastreams, with the intended primary use case in multidimensional photoelectron spectroscopy using time-of-flight instruments. -# Table of Contents -[Installation](#installation) - - [For Users (pip)](#for-users-pip) - - [For Contributors (pip)](#for-contributors-pip) - - [For Maintainers (poetry)](#for-maintainers-poetry) +It builds on [Dask](https://www.dask.org/) dataframes, where each column represents a multidimensional "coordinate" such as position, time-of-flight, pump-probe delay etc., and each entry represents one electron. The `SedProcessor` class provides a single user entry point, and provides functions for handling various workflows for coordinate transformation, e.g. corrections and calibrations. -# Installation +Furthermore, "sed-processor" provides fast and parallelized binning routines to compute multidimensional histograms from the processed dataframes in a delayed fashion, thus reducing requirements on cpu power and memory consumption. + +Finally, in contains several export routines, including export into the [NeXus](https://www.nexusformat.org/) format with rich and standardized metadata annotation. -## For Users (pip) +# Installation -### Prerequisites +## Prerequisites - Python 3.8+ - pip -### Steps +## Steps - Create a new virtual environment using either venv, pyenv, conda, etc. See below for an example. ```bash @@ -58,75 +56,29 @@ python -m ipykernel install --user --name=sed_kernel pip install sed-processor ``` -## For Contributors (pip) - -### Prerequisites -- Git -- Python 3.8+ -- pip - -### Steps -1. Clone the repository: - -```bash -git clone https://github.com/OpenCOMPES/sed.git -cd sed -``` - -2. Create and activate a virtual environment: - -```bash -# Create a virtual environment -python -m venv .sed-dev +# Documentation +Comprehensive documentation including several workflow examples can be found here: +https://opencompes.github.io/docs/sed/latest/ -# Activate the virtual environment -# On macOS/Linux -source .sed-dev/bin/activate -# On Windows -.sed-dev\Scripts\activate -``` +# Contributing +Users are welcome to contribute to the development of **sed-processor**. Information how to contribute, including how to install developer versions can be found in the [documentation](https://opencompes.github.io/docs/sed/latest/misc/contribution.html) -3. Install the repository in editable mode with all dependencies: - -```bash -pip install -e .[all] -``` - -Now you have the development version of `sed` installed in your local environment. Feel free to make changes and submit pull requests. - -## For Maintainers (poetry) - -### Prerequisites -- Poetry: [Poetry Installation](https://python-poetry.org/docs/#installation) - -### Steps -- Create a virtual environment by typing: - -```bash -poetry shell -``` - -- A new shell will be spawned with the new environment activated. - -- Install the dependencies from the `pyproject.toml` by typing: - -```bash -poetry install --with dev, docs -``` +We would like to thank our contributors! -- If you wish to use the virtual environment created by Poetry to work in a Jupyter notebook, you first need to install the optional notebook dependencies and then create a Jupyter kernel for that. +[![Contributors](https://contrib.rocks/image?repo=OpenCOMPES/sed)](https://github.com/OpenCOMPES/sed/graphs/contributors) - - Install the optional dependencies: - ```bash - poetry install -E notebook - ``` +## License - - Make sure to run the command below within your virtual environment (`poetry run` ensures this) by typing: +sed-processor is licenced under the MIT license - ```bash - poetry run ipython kernel install --user --name=sed_poetry - ``` +Copyright (c) 2022-2024 OpenCOMPES - - The new kernel will now be available in your Jupyter kernels list. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/conf.py b/docs/conf.py index bd4c1396..68ec181c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,8 +25,9 @@ def _get_project_meta(): return tomlkit.parse(file_contents)["tool"]["poetry"] +# -- Project information ----------------------------------------------------- pkg_meta = _get_project_meta() -project = str(pkg_meta["name"]) +project = "SED" copyright = "2024, OpenCOMPES team" author = "OpenCOMPES team" @@ -41,12 +42,12 @@ def _get_project_meta(): # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx_rtd_theme", "sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.coverage", "sphinx.ext.autosummary", + "sphinx.ext.viewcode", "sphinx.ext.coverage", "sphinx_autodoc_typehints", "bokeh.sphinxext.bokeh_autodoc", @@ -56,6 +57,8 @@ def _get_project_meta(): ] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] + autoclass_content = "class" autodoc_member_order = "bysource" @@ -98,7 +101,28 @@ def _get_project_meta(): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "pydata_sphinx_theme" + +html_theme_options = { + "github_url": "https://github.com/OpenCOMPES/sed", + "primary_sidebar_end": ["indices.html"], + "navbar_center": ["version-switcher", "navbar-nav"], + "show_nav_level": 2, + "show_version_warning_banner": True, + # maybe better to use _static/switcher.json on github pages link instead of the following + "switcher": { + "json_url": "https://raw.githubusercontent.com/OpenCOMPES/docs/main/sed/switcher.json", + "version_match": version, + }, + "content_footer_items": ["last-updated"], +} + +html_context = { + "github_user": "OpenCOMPES", + "github_repo": "sed", + "github_version": "main", + "doc_path": "docs", +} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/getting_started.rst b/docs/getting_started.rst deleted file mode 100644 index fba01724..00000000 --- a/docs/getting_started.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. include:: ../README.md - :parser: myst_parser.sphinx_ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..afeefb05 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,54 @@ +--- +myst: + html_meta: + "description lang=en": | + Top-level documentation for sed, with links to the rest + of the site.. +html_theme.sidebar_secondary.remove: true +--- + +# SED documentation + +SED (Single Event Data Frame) is a collection of routines and utilities to handle photoelectron resolved datastreams. +It features lazy evaluation of dataframe processing using dask, numba-accelerated multi-dimensional binning, calibration and correction for trARPES (Time- and angle-resolved photoemission spectroscopy) datasets. +The package ensures provenance and FAIR data through metadata tracking, usage of the community defined NeXus format. + +## User guide + +We introduce different functionalities of the package by several step-by-step use guides: + +```{toctree} +:maxdepth: 2 + +user_guide/index + +``` + +## Examples + +Several example notebooks to demonstrate the functionality of SED for end-to-end data analysis workflows. + +```{toctree} +:maxdepth: 2 + +workflows/index +``` + +## API + +```{toctree} +:maxdepth: 2 + +sed/api +``` + + +## Community and contribution guide + +Information about the community behind this theme and how you can contribute. + +```{toctree} +:maxdepth: 2 + +misc/contribution +``` diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index b61dcb31..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,44 +0,0 @@ -Single-Event DataFrame (SED) documentation -================================================ - -.. toctree:: - :maxdepth: 1 - :caption: Getting Started - - getting_started - tutorial/1_binning_fake_data - tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data - tutorial/3_metadata_collection_and_export_to_NeXus - tutorial/4_hextof_workflow.ipynb - tutorial/6_binning_with_time-stamped_data - tutorial/7_correcting_orthorhombic_symmetry - tutorial/8_jittering_tutorial - -.. toctree:: - :maxdepth: 1 - :caption: SED Core Modules - - sed/core - sed/dfops - sed/loader - sed/binning - sed/calibrator - sed/dataset - sed/diagnostic - sed/io - sed/metadata - sed/config - -.. toctree:: - :maxdepth: 2 - :caption: Contributing - - misc/contributing - misc/maintain - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/misc/contributing.rst b/docs/misc/contributing.rst index ab12e517..7aab314a 100644 --- a/docs/misc/contributing.rst +++ b/docs/misc/contributing.rst @@ -8,7 +8,6 @@ Whether you are a beamline scientist hoping to create a loader for your data, or This guide will walk you through the process of setting up your development environment, and the workflow for contributing to the project. -.. contents:: Getting Started =============== diff --git a/docs/misc/contribution.md b/docs/misc/contribution.md new file mode 100644 index 00000000..88d7b568 --- /dev/null +++ b/docs/misc/contribution.md @@ -0,0 +1,8 @@ +# Development + +```{toctree} +:maxdepth: 2 + +contributing +maintain +``` diff --git a/docs/misc/maintain.rst b/docs/misc/maintain.rst index 486f426a..cf8d0a93 100644 --- a/docs/misc/maintain.rst +++ b/docs/misc/maintain.rst @@ -140,7 +140,7 @@ To create a release, follow these steps: c. **If you don't see update on PyPI:** - Visit the GitHub Actions page and monitor the Release workflow (https://github.com/OpenCOMPES/sed/actions/workflows/release.yml). - - Check if errors occurred. + - Check if errors occurred during the release process. **Understanding the Release Workflow** diff --git a/docs/sed/api.rst b/docs/sed/api.rst new file mode 100644 index 00000000..fffcc53f --- /dev/null +++ b/docs/sed/api.rst @@ -0,0 +1,17 @@ +====== +API +====== + +.. toctree:: + :maxdepth: 1 + + core + dfops + loader + binning + calibrator + dataset + diagnostic + io + metadata + config diff --git a/docs/sed/config.rst b/docs/sed/config.rst index c5eac7a2..485bd7ea 100644 --- a/docs/sed/config.rst +++ b/docs/sed/config.rst @@ -1,17 +1,4 @@ Config -=================================================== -The config module contains a mechanics to collect configuration parameters from various sources and configuration files, and to combine them in a hierarchical manner into a single, consistent configuration dictionary. -It will load an (optional) provided config file, or alternatively use a passed python dictionary as initial config dictionary, and subsequently look for the following additional config files to load: - -* ``folder_config``: A config file of name :file:`sed_config.yaml` in the current working directory. This is mostly intended to pass calibration parameters of the workflow between different notebook instances. -* ``user_config``: A config file provided by the user, stored as :file:`.config/sed/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. -* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/sed/config.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/sed/config.yaml` on Windows. This should provide all necessary default parameters for using the sed processor with a given setup. For an example for an mpes setup, see :ref:`example_config` -* ``default_config``: The default configuration shipped with the package. Typically, all parameters here should be overwritten by any of the other configuration files. - -The config mechanism returns the combined dictionary, and reports the loaded configuration files. In order to disable or overwrite any of the configuration files, they can be also given as optional parameters (path to a file, or python dictionary). - - -API *************************************************** .. automodule:: sed.core.config :members: @@ -19,21 +6,3 @@ API .. _example_config: - -Default configuration settings -*************************************************** - -.. literalinclude:: ../../sed/config/default.yaml - :language: yaml - -Example configuration file for mpes (METIS momentum microscope at FHI-Berlin) -********************************************************************************* - -.. literalinclude:: ../../sed/config/mpes_example_config.yaml - :language: yaml - -Example configuration file for flash (HEXTOF momentum microscope at FLASH, Desy) -********************************************************************************* - -.. literalinclude:: ../../sed/config/flash_example_config.yaml - :language: yaml diff --git a/docs/sed/dataset.rst b/docs/sed/dataset.rst index 05bd94c0..b0147464 100644 --- a/docs/sed/dataset.rst +++ b/docs/sed/dataset.rst @@ -13,14 +13,12 @@ Getting datasets import os from sed.dataset import dataset -Get -~~~ +get() +^^^^^ The “get” just needs the data name, but another root_dir can be provided. -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Try to interrupt the download process and restart to see that it continues the download from where it stopped -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -39,14 +37,12 @@ Try to interrupt the download process and restart to see that it continues the d Download complete. Not providing “remove_zip” at all will by default delete the zip file after extraction -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python dataset.get("WSe2") Setting the “use_existing” keyword to False allows to download the data in another location. Default is to use existing data -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -62,10 +58,8 @@ Setting the “use_existing” keyword to False allows to download the data in a Interrupting extraction has similar behavior to download and just continues from where it stopped. -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' Or if user deletes the extracted documents, it re-extracts from zip file -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -90,11 +84,12 @@ Or if user deletes the extracted documents, it re-extracts from zip file WSe2 data extracted successfully. +remove() +^^^^^^^^ + “remove” allows removal of some or all instances of existing data -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This would remove only one of the two existing paths -'''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -105,7 +100,6 @@ This would remove only one of the two existing paths Removed <user_path>/datasets/WSe2 This removes all instances, if any present -'''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -116,10 +110,9 @@ This removes all instances, if any present WSe2 data is not present. Attributes useful for user -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ All available datasets after looking at module, user and folder levels -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -130,7 +123,6 @@ All available datasets after looking at module, user and folder levels ['WSe2', 'TaS2', 'Gd_W110'] The dir and subdirs where data is located -''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -150,7 +142,6 @@ The dir and subdirs where data is located '<user_path>/datasets/WSe2/energycal_2019_01_08'] Existing locations where data is present -'''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -164,11 +155,12 @@ Existing locations where data is present Example of adding custom datasets --------------------------------- +DatasetsManager +^^^^^^^^^^^^^^^ + Allows to add or remove datasets in json file at any level (module, user, folder). -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' Looks at all levels to give the available datasets -'''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -176,10 +168,8 @@ Looks at all levels to give the available datasets from sed.dataset import DatasetsManager We add a new dataset to both folder and user levels -''''''''''''''''''''''''''''''''''''''''''''''''''' This dataset also has “rearrange_files” set to True, which takes all files in subfolders and puts them in the main dataset specific directory -''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -198,7 +188,6 @@ This dataset also has “rearrange_files” set to True, which takes all files i Added Example dataset to user datasets.json datasets.json should be available in execution folder after this -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -210,7 +199,6 @@ datasets.json should be available in execution folder after this ['Example', 'WSe2', 'TaS2', 'Gd_W110'] This will remove the Example dataset from the user json file -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -221,7 +209,6 @@ This will remove the Example dataset from the user json file Removed Example dataset from user datasets.json Adding dataset that already exists will give an error. Likewise, removing one that doesn’t exist -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -234,7 +221,6 @@ Adding dataset that already exists will give an error. Likewise, removing one th Now that dataset.json with Example exists in current dir, lets try to fetch it -'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. code:: python @@ -272,7 +258,6 @@ Now that dataset.json with Example exists in current dir, lets try to fetch it [] lets download to another location -''''''''''''''''''''''''''''''''' .. code:: python @@ -300,7 +285,6 @@ lets download to another location Rearranging complete. we can remove one instance -'''''''''''''''''''''''''' .. code:: python @@ -332,7 +316,7 @@ we can remove one instance ['<user_path>/datasets/Example'] Default datasets.json ------------------------- +--------------------- .. literalinclude:: ../../sed/dataset/datasets.json :language: json diff --git a/docs/sed/loader.rst b/docs/sed/loader.rst index 87fdebdb..468f34e4 100644 --- a/docs/sed/loader.rst +++ b/docs/sed/loader.rst @@ -10,6 +10,7 @@ Loader Interface :undoc-members: .. _base_loader: + Abstract BaseLoader ################################################### .. automodule:: sed.loader.base.loader diff --git a/docs/user_guide/advanced_topics.md b/docs/user_guide/advanced_topics.md new file mode 100644 index 00000000..ebd773c8 --- /dev/null +++ b/docs/user_guide/advanced_topics.md @@ -0,0 +1,6 @@ +```{toctree} +:maxdepth: 1 +../tutorial/6_binning_with_time-stamped_data +../tutorial/7_correcting_orthorhombic_symmetry +../tutorial/8_jittering_tutorial +``` diff --git a/docs/user_guide/config.md b/docs/user_guide/config.md new file mode 100644 index 00000000..7fb62f7d --- /dev/null +++ b/docs/user_guide/config.md @@ -0,0 +1,29 @@ +# Configuration + +The config module contains a mechanism to collect configuration parameters from various sources and configuration files, and to combine them in a hierarchical manner into a single, consistent configuration dictionary. +It will load an (optional) provided config file, or alternatively use a passed python dictionary as initial config dictionary, and subsequently look for the following additional config files to load: + +* ``folder_config``: A config file of name :file:`sed_config.yaml` in the current working directory. This is mostly intended to pass calibration parameters of the workflow between different notebook instances. +* ``user_config``: A config file provided by the user, stored as :file:`.sed/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. +* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/sed/config.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/sed/config.yaml` on Windows. This should provide all necessary default parameters for using the sed processor with a given setup. For an example for an mpes setup, see :ref:`example_config` +* ``default_config``: The default configuration shipped with the package. Typically, all parameters here should be overwritten by any of the other configuration files. + +The config mechanism returns the combined dictionary, and reports the loaded configuration files. In order to disable or overwrite any of the configuration files, they can be also given as optional parameters (path to a file, or python dictionary). + +## Default configuration settings + +```{literalinclude} ../../sed/config/default.yaml +:language: yaml +``` + +## Example configuration file for mpes (METIS momentum microscope at FHI-Berlin) + +```{literalinclude} ../../sed/config/mpes_example_config.yaml +:language: yaml +``` + +## Example configuration file for flash (HEXTOF momentum microscope at FLASH, Desy) + +```{literalinclude} ../../sed/config/flash_example_config.yaml +:language: yaml +``` diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md new file mode 100644 index 00000000..16d36aaf --- /dev/null +++ b/docs/user_guide/index.md @@ -0,0 +1,29 @@ +--- +myst: + html_meta: + "description lang=en": | + Documentation for users. +--- +# User Guide + +## Installing SED +```{toctree} +:maxdepth: 1 +installation +``` + +## Basic concepts +```{toctree} +:maxdepth: 1 +../tutorial/1_binning_fake_data +../tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data +../tutorial/3_metadata_collection_and_export_to_NeXus +config +``` + +## Advanced Topics + +```{toctree} +:maxdepth: 1 +advanced_topics +``` diff --git a/docs/user_guide/installation.md b/docs/user_guide/installation.md new file mode 100644 index 00000000..8c3f44dd --- /dev/null +++ b/docs/user_guide/installation.md @@ -0,0 +1,74 @@ +# Installation + +```{attention} +Requires Python 3.9+ and pip installed. +``` + +- Create a new virtual environment using either venv, pyenv, conda, etc. See below for an example. + +```bash +python -m venv .sed-venv +``` + +- Activate your environment: + +```bash +# On macOS/Linux +source .sed-venv/bin/activate + +# On Windows +.sed-venv\Scripts\activate +``` + +- Install `sed`, distributed as `sed-processor` on PyPI: + +```bash +pip install sed-processor[all] +``` + +- If you do not use Jupyter Notebook or Jupyter Lab, you can skip the installing those dependencies: + +```bash +pip install sed-processor +``` + +```{note} +If you intend to work with Jupyter notebooks, it is helpful to install a Jupyter kernel for your environment. This can be done, once your environment is activated, by typing: +```bash +python -m ipykernel install --user --name=sed_kernel +``` + +# Development version + +```{attention} +Requires Git, Python 3.9+ and pip installed. +``` + +1. Clone the repository: + +```bash +git clone https://github.com/OpenCOMPES/sed.git +cd sed +``` + +2. Create and activate a virtual environment: + +```bash +# Create a virtual environment +python -m venv .sed-dev + +# Activate the virtual environment +# On macOS/Linux +source .sed-dev/bin/activate + +# On Windows +.sed-dev\Scripts\activate +``` + +3. Install the repository in editable mode with all dependencies: + +```bash +pip install -e .[all] +``` + +Now you have the development version of `sed` installed in your local environment. Feel free to make changes and submit pull requests. diff --git a/docs/workflows/index.md b/docs/workflows/index.md new file mode 100644 index 00000000..f056d133 --- /dev/null +++ b/docs/workflows/index.md @@ -0,0 +1,11 @@ +--- +myst: + html_meta: + "description lang=en": | + Workflows showcasing different SED loaders and methods +--- +# Workflows + +```{toctree} +../tutorial/4_hextof_workflow +``` diff --git a/poetry.lock b/poetry.lock index acf27ecf..fda5d737 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiofiles" @@ -882,37 +882,37 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest- [[package]] name = "debugpy" -version = "1.8.7" +version = "1.8.8" description = "An implementation of the Debug Adapter Protocol for Python" optional = false python-versions = ">=3.8" files = [ - {file = "debugpy-1.8.7-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:95fe04a573b8b22896c404365e03f4eda0ce0ba135b7667a1e57bd079793b96b"}, - {file = "debugpy-1.8.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:628a11f4b295ffb4141d8242a9bb52b77ad4a63a2ad19217a93be0f77f2c28c9"}, - {file = "debugpy-1.8.7-cp310-cp310-win32.whl", hash = "sha256:85ce9c1d0eebf622f86cc68618ad64bf66c4fc3197d88f74bb695a416837dd55"}, - {file = "debugpy-1.8.7-cp310-cp310-win_amd64.whl", hash = "sha256:29e1571c276d643757ea126d014abda081eb5ea4c851628b33de0c2b6245b037"}, - {file = "debugpy-1.8.7-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:caf528ff9e7308b74a1749c183d6808ffbedbb9fb6af78b033c28974d9b8831f"}, - {file = "debugpy-1.8.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cba1d078cf2e1e0b8402e6bda528bf8fda7ccd158c3dba6c012b7897747c41a0"}, - {file = "debugpy-1.8.7-cp311-cp311-win32.whl", hash = "sha256:171899588bcd412151e593bd40d9907133a7622cd6ecdbdb75f89d1551df13c2"}, - {file = "debugpy-1.8.7-cp311-cp311-win_amd64.whl", hash = "sha256:6e1c4ffb0c79f66e89dfd97944f335880f0d50ad29525dc792785384923e2211"}, - {file = "debugpy-1.8.7-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:4d27d842311353ede0ad572600c62e4bcd74f458ee01ab0dd3a1a4457e7e3706"}, - {file = "debugpy-1.8.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:703c1fd62ae0356e194f3e7b7a92acd931f71fe81c4b3be2c17a7b8a4b546ec2"}, - {file = "debugpy-1.8.7-cp312-cp312-win32.whl", hash = "sha256:2f729228430ef191c1e4df72a75ac94e9bf77413ce5f3f900018712c9da0aaca"}, - {file = "debugpy-1.8.7-cp312-cp312-win_amd64.whl", hash = "sha256:45c30aaefb3e1975e8a0258f5bbd26cd40cde9bfe71e9e5a7ac82e79bad64e39"}, - {file = "debugpy-1.8.7-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:d050a1ec7e925f514f0f6594a1e522580317da31fbda1af71d1530d6ea1f2b40"}, - {file = "debugpy-1.8.7-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2f4349a28e3228a42958f8ddaa6333d6f8282d5edaea456070e48609c5983b7"}, - {file = "debugpy-1.8.7-cp313-cp313-win32.whl", hash = "sha256:11ad72eb9ddb436afb8337891a986302e14944f0f755fd94e90d0d71e9100bba"}, - {file = "debugpy-1.8.7-cp313-cp313-win_amd64.whl", hash = "sha256:2efb84d6789352d7950b03d7f866e6d180284bc02c7e12cb37b489b7083d81aa"}, - {file = "debugpy-1.8.7-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:4b908291a1d051ef3331484de8e959ef3e66f12b5e610c203b5b75d2725613a7"}, - {file = "debugpy-1.8.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da8df5b89a41f1fd31503b179d0a84a5fdb752dddd5b5388dbd1ae23cda31ce9"}, - {file = "debugpy-1.8.7-cp38-cp38-win32.whl", hash = "sha256:b12515e04720e9e5c2216cc7086d0edadf25d7ab7e3564ec8b4521cf111b4f8c"}, - {file = "debugpy-1.8.7-cp38-cp38-win_amd64.whl", hash = "sha256:93176e7672551cb5281577cdb62c63aadc87ec036f0c6a486f0ded337c504596"}, - {file = "debugpy-1.8.7-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:90d93e4f2db442f8222dec5ec55ccfc8005821028982f1968ebf551d32b28907"}, - {file = "debugpy-1.8.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6db2a370e2700557a976eaadb16243ec9c91bd46f1b3bb15376d7aaa7632c81"}, - {file = "debugpy-1.8.7-cp39-cp39-win32.whl", hash = "sha256:a6cf2510740e0c0b4a40330640e4b454f928c7b99b0c9dbf48b11efba08a8cda"}, - {file = "debugpy-1.8.7-cp39-cp39-win_amd64.whl", hash = "sha256:6a9d9d6d31846d8e34f52987ee0f1a904c7baa4912bf4843ab39dadf9b8f3e0d"}, - {file = "debugpy-1.8.7-py2.py3-none-any.whl", hash = "sha256:57b00de1c8d2c84a61b90880f7e5b6deaf4c312ecbde3a0e8912f2a56c4ac9ae"}, - {file = "debugpy-1.8.7.zip", hash = "sha256:18b8f731ed3e2e1df8e9cdaa23fb1fc9c24e570cd0081625308ec51c82efe42e"}, + {file = "debugpy-1.8.8-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:e59b1607c51b71545cb3496876544f7186a7a27c00b436a62f285603cc68d1c6"}, + {file = "debugpy-1.8.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6531d952b565b7cb2fbd1ef5df3d333cf160b44f37547a4e7cf73666aca5d8d"}, + {file = "debugpy-1.8.8-cp310-cp310-win32.whl", hash = "sha256:b01f4a5e5c5fb1d34f4ccba99a20ed01eabc45a4684f4948b5db17a319dfb23f"}, + {file = "debugpy-1.8.8-cp310-cp310-win_amd64.whl", hash = "sha256:535f4fb1c024ddca5913bb0eb17880c8f24ba28aa2c225059db145ee557035e9"}, + {file = "debugpy-1.8.8-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:c399023146e40ae373753a58d1be0a98bf6397fadc737b97ad612886b53df318"}, + {file = "debugpy-1.8.8-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09cc7b162586ea2171eea055985da2702b0723f6f907a423c9b2da5996ad67ba"}, + {file = "debugpy-1.8.8-cp311-cp311-win32.whl", hash = "sha256:eea8821d998ebeb02f0625dd0d76839ddde8cbf8152ebbe289dd7acf2cdc6b98"}, + {file = "debugpy-1.8.8-cp311-cp311-win_amd64.whl", hash = "sha256:d4483836da2a533f4b1454dffc9f668096ac0433de855f0c22cdce8c9f7e10c4"}, + {file = "debugpy-1.8.8-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:0cc94186340be87b9ac5a707184ec8f36547fb66636d1029ff4f1cc020e53996"}, + {file = "debugpy-1.8.8-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64674e95916e53c2e9540a056e5f489e0ad4872645399d778f7c598eacb7b7f9"}, + {file = "debugpy-1.8.8-cp312-cp312-win32.whl", hash = "sha256:5c6e885dbf12015aed73770f29dec7023cb310d0dc2ba8bfbeb5c8e43f80edc9"}, + {file = "debugpy-1.8.8-cp312-cp312-win_amd64.whl", hash = "sha256:19ffbd84e757a6ca0113574d1bf5a2298b3947320a3e9d7d8dc3377f02d9f864"}, + {file = "debugpy-1.8.8-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:705cd123a773d184860ed8dae99becd879dfec361098edbefb5fc0d3683eb804"}, + {file = "debugpy-1.8.8-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:890fd16803f50aa9cb1a9b9b25b5ec321656dd6b78157c74283de241993d086f"}, + {file = "debugpy-1.8.8-cp313-cp313-win32.whl", hash = "sha256:90244598214bbe704aa47556ec591d2f9869ff9e042e301a2859c57106649add"}, + {file = "debugpy-1.8.8-cp313-cp313-win_amd64.whl", hash = "sha256:4b93e4832fd4a759a0c465c967214ed0c8a6e8914bced63a28ddb0dd8c5f078b"}, + {file = "debugpy-1.8.8-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:143ef07940aeb8e7316de48f5ed9447644da5203726fca378f3a6952a50a9eae"}, + {file = "debugpy-1.8.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f95651bdcbfd3b27a408869a53fbefcc2bcae13b694daee5f1365b1b83a00113"}, + {file = "debugpy-1.8.8-cp38-cp38-win32.whl", hash = "sha256:26b461123a030e82602a750fb24d7801776aa81cd78404e54ab60e8b5fecdad5"}, + {file = "debugpy-1.8.8-cp38-cp38-win_amd64.whl", hash = "sha256:f3cbf1833e644a3100eadb6120f25be8a532035e8245584c4f7532937edc652a"}, + {file = "debugpy-1.8.8-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:53709d4ec586b525724819dc6af1a7703502f7e06f34ded7157f7b1f963bb854"}, + {file = "debugpy-1.8.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a9c013077a3a0000e83d97cf9cc9328d2b0bbb31f56b0e99ea3662d29d7a6a2"}, + {file = "debugpy-1.8.8-cp39-cp39-win32.whl", hash = "sha256:ffe94dd5e9a6739a75f0b85316dc185560db3e97afa6b215628d1b6a17561cb2"}, + {file = "debugpy-1.8.8-cp39-cp39-win_amd64.whl", hash = "sha256:5c0e5a38c7f9b481bf31277d2f74d2109292179081f11108e668195ef926c0f9"}, + {file = "debugpy-1.8.8-py2.py3-none-any.whl", hash = "sha256:ec684553aba5b4066d4de510859922419febc710df7bba04fe9e7ef3de15d34f"}, + {file = "debugpy-1.8.8.zip", hash = "sha256:e6355385db85cbd666be703a96ab7351bc9e6c61d694893206f8001e22aee091"}, ] [[package]] @@ -3011,13 +3011,13 @@ files = [ [[package]] name = "packaging" -version = "24.1" +version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] [[package]] @@ -4769,13 +4769,13 @@ files = [ [[package]] name = "tqdm" -version = "4.66.6" +version = "4.67.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.66.6-py3-none-any.whl", hash = "sha256:223e8b5359c2efc4b30555531f09e9f2f3589bcd7fdd389271191031b49b7a63"}, - {file = "tqdm-4.66.6.tar.gz", hash = "sha256:4bdd694238bef1485ce839d67967ab50af8f9272aab687c0d7702a01da0be090"}, + {file = "tqdm-4.67.0-py3-none-any.whl", hash = "sha256:0cd8af9d56911acab92182e88d763100d4788bdf421d251616040cc4d44863be"}, + {file = "tqdm-4.67.0.tar.gz", hash = "sha256:fe5a6f95e6fe0b9755e9469b77b9c3cf850048224ecaa8293d7d2d31f97d869a"}, ] [package.dependencies] @@ -4783,6 +4783,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +discord = ["requests"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] @@ -5158,4 +5159,4 @@ notebook = ["ipykernel", "jupyter", "jupyterlab", "jupyterlab-h5web"] [metadata] lock-version = "2.0" python-versions = ">=3.8, <3.11.9" -content-hash = "8b9853a97b3999bbc047d5468b4a33625b52aeb8f7d11b363d40800747fc5967" +content-hash = "3ef2b60e9439e3df820760ce342bbff380d8cf8f0f5a682918b458838347f935" diff --git a/pyproject.toml b/pyproject.toml index f9fd5514..c9854e6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,6 @@ optional = true [tool.poetry.group.docs.dependencies] sphinx = ">=7.1.2" -sphinx-rtd-theme = ">=1.0.0" tomlkit = ">=0.12.0" sphinx-autodoc-typehints = ">=1.17.0" nbsphinx = ">=0.9.3" diff --git a/tutorial/1_binning_fake_data.ipynb b/tutorial/1_binning_fake_data.ipynb index 04370e8e..01e987ae 100644 --- a/tutorial/1_binning_fake_data.ipynb +++ b/tutorial/1_binning_fake_data.ipynb @@ -20,8 +20,6 @@ "metadata": {}, "outputs": [], "source": [ - "import sys\n", - "\n", "import dask\n", "import numpy as np\n", "import pandas as pd\n", @@ -29,8 +27,9 @@ "\n", "import matplotlib.pyplot as plt\n", "\n", - "sys.path.append(\"../\")\n", - "from sed.binning import bin_partition, bin_dataframe" + "from sed.binning import bin_partition, bin_dataframe\n", + "\n", + "%matplotlib widget" ] }, { diff --git a/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb index 7b9cb1e2..6ee3d7c1 100644 --- a/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb +++ b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb @@ -315,13 +315,21 @@ "sp.apply_momentum_calibration()" ] }, + { + "cell_type": "markdown", + "id": "74e7ef75", + "metadata": {}, + "source": [ + "## Energy Correction and Calibration workflow" + ] + }, { "attachments": {}, "cell_type": "markdown", "id": "0bce2388", "metadata": {}, "source": [ - "## Energy Correction (optional)\n", + "### Energy Correction (optional)\n", "The purpose of the energy correction is to correct for any momentum-dependent distortion of the energy axis, e.g. from geometric effects in the flight tube, or from space charge" ] }, @@ -394,7 +402,7 @@ "id": "8b571b4c", "metadata": {}, "source": [ - "## 3. Energy calibration\n", + "### Energy calibration\n", "For calibrating the energy axis, a set of data taken at different bias voltages around the value where the measurement was taken is required." ] },