diff --git a/.github/workflows/build_deploy_docs.yml b/.github/workflows/build_deploy_docs.yml new file mode 100644 index 00000000..718ab5f3 --- /dev/null +++ b/.github/workflows/build_deploy_docs.yml @@ -0,0 +1,97 @@ +name: build and deploy docs to pages +on: + # Triggers the workflow on push but only for the main branch + push: + branches: [ main ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + + # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: 'true' + remove-codeql: 'true' + remove-android: 'true' + remove-docker-images: 'true' + + # Check out repo and set up Python + - name: Check out the repository + uses: actions/checkout@v4 + with: + lfs: true + + # Use cached python and dependencies, install poetry + - name: "Setup Python, Poetry and Dependencies" + uses: packetcoders/action-setup-cache-python-poetry@main + with: + python-version: 3.8 + poetry-version: 1.2.2 + + - name: Install docs and notebook dependencies + run: poetry install -E notebook -E docs + + - name: Install pandoc + run: | + sudo wget https://github.com/jgm/pandoc/releases/download/3.1.8/pandoc-3.1.8-1-amd64.deb + sudo dpkg -i pandoc-3.1.8-1-amd64.deb + + - name: copy tutorial files to docs + run: | + cp -r $GITHUB_WORKSPACE/tutorial $GITHUB_WORKSPACE/docs/ + cp -r $GITHUB_WORKSPACE/sed/config $GITHUB_WORKSPACE/docs/sed + + # To be included later + # - name: Cache docs build + # id: cache-docs + # uses: actions/cache@v3 + # with: + # path: $GITHUB_WORKSPACE/_build + # key: ${{ runner.os }}-docs + + - name: download WSe2 data + # if: steps.cache-primes.outputs.cache-hit != 'true' + run: | + cd $GITHUB_WORKSPACE/docs/tutorial + curl -L --output ./WSe2.zip https://zenodo.org/record/6369728/files/WSe2.zip + unzip -o ./WSe2.zip -d . + + - name: build Sphinx docs + run: poetry run sphinx-build -b html $GITHUB_WORKSPACE/docs $GITHUB_WORKSPACE/_build + + - name: Setup Pages + uses: actions/configure-pages@v3 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: '_build' + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index eecb9da2..1a97ae54 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -11,14 +11,14 @@ jobs: - uses: actions/checkout@v3 with: lfs: true - + # Use cached python and dependencies, install poetry - name: "Setup Python, Poetry and Dependencies" uses: packetcoders/action-setup-cache-python-poetry@main with: python-version: 3.8 poetry-version: 1.2.2 - + # Linting steps, excute all linters even if one fails - name: pycodestyle run: diff --git a/.github/workflows/update_dependencies.yml b/.github/workflows/update_dependencies.yml index c7f08d4b..6aecf8b3 100644 --- a/.github/workflows/update_dependencies.yml +++ b/.github/workflows/update_dependencies.yml @@ -41,10 +41,6 @@ jobs: echo "$UPDATE_OUTPUT" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - name: Export requirements.txt - run: | - poetry export --without-hashes --format=requirements.txt -o docs/requirements.txt -E docs -E notebook - - name: Obtain git status id: status run: | diff --git a/.gitignore b/.gitignore index c7d4c380..5cc3ff6b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.nxs *.nx *.nxs +*.zip # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index 27a88b21..5ea5c0f6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # sed -[![Documentation Status](https://readthedocs.org/projects/sed/badge/?version=latest)](https://sed.readthedocs.io/en/latest/?badge=latest) +[![Documentation Status](https://github.com/OpenCOMPES/sed/actions/workflows/build_deploy_docs.yml/badge.svg)](https://opencompes.github.io/sed/) ![](https://github.com/OpenCOMPES/sed/actions/workflows/linting.yml/badge.svg?branch=main) ![](https://github.com/OpenCOMPES/sed/actions/workflows/testing_multiversion.yml/badge.svg?branch=main) ![](https://img.shields.io/pypi/pyversions/sedprocessor) diff --git a/tutorial/Binning Benchmarks.ipynb b/benchmarks/Binning Benchmarks.ipynb similarity index 100% rename from tutorial/Binning Benchmarks.ipynb rename to benchmarks/Binning Benchmarks.ipynb diff --git a/tutorial/2 - Binning of example time-resolved ARPES data.ipynb b/benchmarks/mpes_sed_benchmarks.ipynb similarity index 95% rename from tutorial/2 - Binning of example time-resolved ARPES data.ipynb rename to benchmarks/mpes_sed_benchmarks.ipynb index 92982d31..605ad175 100644 --- a/tutorial/2 - Binning of example time-resolved ARPES data.ipynb +++ b/benchmarks/mpes_sed_benchmarks.ipynb @@ -7,7 +7,7 @@ "tags": [] }, "source": [ - "# Binning example time-resolved ARPES data stored on Zenode\n", + "# Binning example time-resolved ARPES data stored on Zenodo\n", "In this example, we pull some time-resolved ARPES data from Zenodo, and generate a dask dataframe using the methods of the mpes package. It requires the mpes package to be installed, in addition to the sed package.\n", "For performance reasons, best store the data on a locally attached storage (no network drive)." ] @@ -25,6 +25,7 @@ "\n", "import matplotlib.pyplot as plt\n", "from mpes import fprocessing as fp\n", + "\n", "import os\n", "import shutil\n", "\n", @@ -37,7 +38,7 @@ "id": "42a6afaa-17dd-4637-ba75-a28c4ead1adf", "metadata": {}, "source": [ - "# Load Data" + "## Load Data" ] }, { @@ -73,7 +74,7 @@ "id": "6902fd56-1456-4da6-83a4-0f3f6b831eb6", "metadata": {}, "source": [ - "# Define the binning range" + "## Define the binning range" ] }, { @@ -104,7 +105,7 @@ "id": "01066d40-010a-490b-9033-7339e5a21b26", "metadata": {}, "source": [ - "# compute distributed binning on the partitioned dask dataframe\n", + "## compute distributed binning on the partitioned dask dataframe\n", "We generated 100 dataframe partiions from the 100 files in the dataset, which we will bin parallelly with the dataframe binning function into a 3D grid" ] }, @@ -141,7 +142,7 @@ "id": "4a3eaf0e", "metadata": {}, "source": [ - "# Compare to MPES binning" + "## Compare to MPES binning" ] }, { @@ -170,7 +171,7 @@ "id": "e3398aac", "metadata": {}, "source": [ - "# Test the class and the histogram function" + "## Test the class and the histogram function" ] }, { diff --git a/docs/conf.py b/docs/conf.py index fa9e0dc6..e1bc1bd0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,12 +10,14 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import tomlkit import os import sys sys.path.insert(0, os.path.abspath('..')) -import tomlkit # -- Project information ----------------------------------------------------- + + def _get_project_meta(): with open('../pyproject.toml') as pyproject: file_contents = pyproject.read() @@ -25,7 +27,7 @@ def _get_project_meta(): pkg_meta = _get_project_meta() project = str(pkg_meta['name']) -copyright = '2022, OpenCOMPES team' +copyright = '2022, OpenCOMPES team' author = 'OpenCOMPES team' # The short X.Y version @@ -38,9 +40,11 @@ def _get_project_meta(): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx_rtd_theme','sphinx.ext.autodoc','sphinx.ext.napoleon', -'sphinx.ext.todo','sphinx.ext.coverage','sphinx.ext.autosummary', -'sphinx.ext.coverage','sphinx_autodoc_typehints'] +extensions = ['sphinx_rtd_theme', 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', + 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.autosummary', + 'sphinx.ext.coverage', 'sphinx_autodoc_typehints', "bokeh.sphinxext.bokeh_autodoc", + "bokeh.sphinxext.bokeh_plot", 'nbsphinx'] + autoclass_content = 'class' autodoc_member_order = 'bysource' @@ -61,7 +65,6 @@ def _get_project_meta(): } - # Set `typing.TYPE_CHECKING` to `True`: # https://pypi.org/project/sphinx-autodoc-typehints/ napoleon_use_param = True @@ -90,4 +93,4 @@ def _get_project_meta(): # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file +html_static_path = ['_static'] diff --git a/docs/examples/example.rst b/docs/examples/example.rst deleted file mode 100644 index 495d503b..00000000 --- a/docs/examples/example.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use notebook to fill -=================================================== \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index c73bb857..6d8755b6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,11 +17,13 @@ Single-Event DataFrame (SED) documentation sed/config .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :numbered: :caption: Examples - examples/example + tutorial/1_binning_fake_data.ipynb + tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb + tutorial/3_metadata_collection_and_export_to_NeXus.ipynb .. toctree:: :maxdepth: 2 diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index b7810b69..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,235 +0,0 @@ -alabaster==0.7.13 ; python_version >= "3.8" and python_version < "3.11" -anyio==4.0.0 ; python_version >= "3.8" and python_version < "3.11" -appnope==0.1.3 ; python_version >= "3.8" and python_version < "3.11" and (platform_system == "Darwin" or sys_platform == "darwin") -argon2-cffi-bindings==21.2.0 ; python_version >= "3.8" and python_version < "3.11" -argon2-cffi==23.1.0 ; python_version >= "3.8" and python_version < "3.11" -arrow==1.3.0 ; python_version >= "3.8" and python_version < "3.11" -asciitree==0.3.3 ; python_version >= "3.8" and python_version < "3.11" and (platform_machine == "x86_64" or platform_machine == "AMD64") -ase==3.22.1 ; python_version >= "3.8" and python_version < "3.11" -asteval==0.9.31 ; python_version >= "3.8" and python_version < "3.11" -astropy==5.2.2 ; python_version >= "3.8" and python_version < "3.11" -asttokens==2.4.1 ; python_version >= "3.8" and python_version < "3.11" -async-lru==2.0.4 ; python_version >= "3.8" and python_version < "3.11" -attrs==23.1.0 ; python_version >= "3.8" and python_version < "3.11" -babel==2.13.1 ; python_version >= "3.8" and python_version < "3.11" -backcall==0.2.0 ; python_version >= "3.8" and python_version < "3.11" -backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9" -beautifulsoup4==4.12.2 ; python_version >= "3.8" and python_version < "3.11" -bleach==6.1.0 ; python_version >= "3.8" and python_version < "3.11" -bokeh==2.4.3 ; python_version >= "3.8" and python_version < "3.11" -cattrs==23.1.2 ; python_version >= "3.8" and python_version < "3.11" -certifi==2023.7.22 ; python_version >= "3.8" and python_version < "3.11" -cffi==1.16.0 ; python_version >= "3.8" and python_version < "3.11" -charset-normalizer==3.3.2 ; python_version >= "3.8" and python_version < "3.11" -click==8.1.7 ; python_version >= "3.8" and python_version < "3.11" -cloudpickle==3.0.0 ; python_version >= "3.8" and python_version < "3.11" -colorama==0.4.6 ; python_version >= "3.8" and python_version < "3.11" and (platform_system == "Windows" or sys_platform == "win32") -comm==0.1.4 ; python_version >= "3.8" and python_version < "3.11" -commonmark==0.9.1 ; python_version >= "3.8" and python_version < "3.11" -contourpy==1.1.1 ; python_version >= "3.8" and python_version < "3.11" -cramjam==2.7.0 ; python_version >= "3.8" and python_version < "3.11" -cycler==0.12.1 ; python_version >= "3.8" and python_version < "3.11" -dask==2022.12.1 ; python_version >= "3.8" and python_version < "3.11" -dask[array]==2022.12.1 ; python_version >= "3.8" and python_version < "3.11" -debugpy==1.8.0 ; python_version >= "3.8" and python_version < "3.11" -decorator==5.1.1 ; python_version >= "3.8" and python_version < "3.11" -defusedxml==0.7.1 ; python_version >= "3.8" and python_version < "3.11" -diffpy-structure==3.1.0 ; python_version >= "3.8" and python_version < "3.11" -diffsims==0.5.2 ; python_version >= "3.8" and python_version < "3.11" -dill==0.3.7 ; python_version >= "3.8" and python_version < "3.11" -docutils==0.18.1 ; python_version >= "3.8" and python_version < "3.11" -entrypoints==0.4 ; python_version >= "3.8" and python_version < "3.11" -exceptiongroup==1.1.3 ; python_version >= "3.8" and python_version < "3.11" -executing==2.0.1 ; python_version >= "3.8" and python_version < "3.11" -fabio==2023.10.0 ; python_version >= "3.8" and python_version < "3.11" -fastdtw==0.3.4 ; python_version >= "3.8" and python_version < "3.11" -fasteners==0.19 ; python_version >= "3.8" and python_version < "3.11" and (platform_machine == "x86_64" or platform_machine == "AMD64") -fastjsonschema==2.18.1 ; python_version >= "3.8" and python_version < "3.11" -fastparquet==0.8.3 ; python_version >= "3.8" and python_version < "3.11" -flatdict==4.0.1 ; python_version >= "3.8" and python_version < "3.11" -fonttools==4.44.0 ; python_version >= "3.8" and python_version < "3.11" -fqdn==1.5.1 ; python_version >= "3.8" and python_version < "3.11" -fsspec==2023.10.0 ; python_version >= "3.8" and python_version < "3.11" -furo==2023.9.10 ; python_version >= "3.8" and python_version < "3.11" -future==0.18.3 ; python_version >= "3.8" and python_version < "3.11" -gitdb==4.0.11 ; python_version >= "3.8" and python_version < "3.11" -gitpython==3.1.40 ; python_version >= "3.8" and python_version < "3.11" -h5py==3.10.0 ; python_version >= "3.8" and python_version < "3.11" -hdf5plugin==4.2.0 ; python_version >= "3.8" and python_version < "3.11" -hyperspy==1.7.4 ; python_version >= "3.8" and python_version < "3.11" -idna==3.4 ; python_version >= "3.8" and python_version < "3.11" -ifes-apt-tc-data-modeling==0.0.9 ; python_version >= "3.8" and python_version < "3.11" -imageio==2.31.6 ; python_version >= "3.8" and python_version < "3.11" -imagesize==1.4.1 ; python_version >= "3.8" and python_version < "3.11" -importlib-metadata==6.8.0 ; python_version >= "3.8" and python_version < "3.11" -importlib-resources==6.1.0 ; python_version >= "3.8" and python_version < "3.10" -ipykernel==6.26.0 ; python_version >= "3.8" and python_version < "3.11" -ipykernel[notebook]==6.26.0 ; python_version >= "3.8" and python_version < "3.11" -ipympl==0.9.3 ; python_version >= "3.8" and python_version < "3.11" -ipyparallel==8.6.1 ; python_version >= "3.8" and python_version < "3.11" -ipython-genutils==0.2.0 ; python_version >= "3.8" and python_version < "3.11" -ipython==8.12.3 ; python_version >= "3.8" and python_version < "3.11" -ipywidgets==8.1.1 ; python_version >= "3.8" and python_version < "3.11" -isoduration==20.11.0 ; python_version >= "3.8" and python_version < "3.11" -jedi==0.19.1 ; python_version >= "3.8" and python_version < "3.11" -jinja2==3.1.2 ; python_version >= "3.8" and python_version < "3.11" -joblib==1.3.2 ; python_version >= "3.8" and python_version < "3.11" -json5==0.9.14 ; python_version >= "3.8" and python_version < "3.11" -jsonpointer==2.4 ; python_version >= "3.8" and python_version < "3.11" -jsonschema-specifications==2023.7.1 ; python_version >= "3.8" and python_version < "3.11" -jsonschema==4.19.2 ; python_version >= "3.8" and python_version < "3.11" -jsonschema[format-nongpl]==4.19.2 ; python_version >= "3.8" and python_version < "3.11" -jupyter-client==8.5.0 ; python_version >= "3.8" and python_version < "3.11" -jupyter-console==6.6.3 ; python_version >= "3.8" and python_version < "3.11" -jupyter-core==5.5.0 ; python_version >= "3.8" and python_version < "3.11" -jupyter-events==0.8.0 ; python_version >= "3.8" and python_version < "3.11" -jupyter-lsp==2.2.0 ; python_version >= "3.8" and python_version < "3.11" -jupyter-server-terminals==0.4.4 ; python_version >= "3.8" and python_version < "3.11" -jupyter-server==2.9.1 ; python_version >= "3.8" and python_version < "3.11" -jupyter[notebook]==1.0.0 ; python_version >= "3.8" and python_version < "3.11" -jupyterlab-pygments==0.2.2 ; python_version >= "3.8" and python_version < "3.11" -jupyterlab-server==2.25.0 ; python_version >= "3.8" and python_version < "3.11" -jupyterlab-widgets==3.0.9 ; python_version >= "3.8" and python_version < "3.11" -jupyterlab==4.0.8 ; python_version >= "3.8" and python_version < "3.11" -kikuchipy==0.9.0 ; python_version >= "3.8" and python_version < "3.11" -kiwisolver==1.4.5 ; python_version >= "3.8" and python_version < "3.11" -lark==1.1.8 ; python_version >= "3.8" and python_version < "3.11" -lazy-loader==0.3 ; python_version >= "3.8" and python_version < "3.11" -llvmlite==0.38.1 ; python_version >= "3.8" and python_version < "3.11" -lmfit==1.2.2 ; python_version >= "3.8" and python_version < "3.11" -locket==1.0.0 ; python_version >= "3.8" and python_version < "3.11" -lxml==4.9.3 ; python_version >= "3.8" and python_version < "3.11" -markupsafe==2.1.3 ; python_version >= "3.8" and python_version < "3.11" -matplotlib-inline==0.1.6 ; python_version >= "3.8" and python_version < "3.11" -matplotlib-scalebar==0.8.1 ; python_version >= "3.8" and python_version < "3.11" -matplotlib==3.7.3 ; python_version >= "3.8" and python_version < "3.11" -mistune==3.0.2 ; python_version >= "3.8" and python_version < "3.11" -mpmath==1.3.0 ; python_version >= "3.8" and python_version < "3.11" -natsort==8.4.0 ; python_version >= "3.8" and python_version < "3.11" -nbclient==0.8.0 ; python_version >= "3.8" and python_version < "3.11" -nbconvert==7.10.0 ; python_version >= "3.8" and python_version < "3.11" -nbformat==5.9.2 ; python_version >= "3.8" and python_version < "3.11" -nest-asyncio==1.5.8 ; python_version >= "3.8" and python_version < "3.11" -networkx==3.1 ; python_version >= "3.8" and python_version < "3.11" -niondata==0.15.3 ; python_version >= "3.8" and python_version < "3.11" -nionswift-io==0.15.1 ; python_version >= "3.8" and python_version < "3.11" -nionswift==0.16.8 ; python_version >= "3.8" and python_version < "3.11" -nionui==0.6.10 ; python_version >= "3.8" and python_version < "3.11" -nionutils==0.4.6 ; python_version >= "3.8" and python_version < "3.11" -notebook-shim==0.2.3 ; python_version >= "3.8" and python_version < "3.11" -notebook==7.0.6 ; python_version >= "3.8" and python_version < "3.11" -numba==0.55.2 ; python_version >= "3.8" and python_version < "3.11" -numcodecs==0.12.1 ; python_version >= "3.8" and python_version < "3.11" and (platform_machine == "x86_64" or platform_machine == "AMD64") -numexpr==2.8.6 ; python_version >= "3.8" and python_version < "3.11" -numpy-quaternion==2022.4.3 ; python_version >= "3.8" and python_version < "3.11" -numpy==1.22.4 ; python_version >= "3.8" and python_version < "3.11" -opencv-python==4.8.0.74 ; python_version >= "3.8" and python_version < "3.11" -orix==0.11.1 ; python_version >= "3.8" and python_version < "3.11" -overrides==7.4.0 ; python_version >= "3.8" and python_version < "3.11" -packaging==23.2 ; python_version >= "3.8" and python_version < "3.11" -pandas==1.5.3 ; python_version >= "3.8" and python_version < "3.11" -pandocfilters==1.5.0 ; python_version >= "3.8" and python_version < "3.11" -parso==0.8.3 ; python_version >= "3.8" and python_version < "3.11" -partd==1.4.1 ; python_version >= "3.8" and python_version < "3.11" -pexpect==4.8.0 ; python_version >= "3.8" and python_version < "3.11" and sys_platform != "win32" -photutils==1.8.0 ; python_version >= "3.8" and python_version < "3.11" -pickleshare==0.7.5 ; python_version >= "3.8" and python_version < "3.11" -pillow==10.0.1 ; python_version >= "3.8" and python_version < "3.11" -pint==0.21.1 ; python_version >= "3.8" and python_version < "3.11" -pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9" -platformdirs==3.11.0 ; python_version >= "3.8" and python_version < "3.11" -ply==3.11 ; python_version >= "3.8" and python_version < "3.11" -pooch==1.8.0 ; python_version >= "3.8" and python_version < "3.11" -prettytable==3.9.0 ; python_version >= "3.8" and python_version < "3.11" -prometheus-client==0.18.0 ; python_version >= "3.8" and python_version < "3.11" -prompt-toolkit==3.0.39 ; python_version >= "3.8" and python_version < "3.11" -psutil==5.9.6 ; python_version >= "3.8" and python_version < "3.11" -ptyprocess==0.7.0 ; python_version >= "3.8" and python_version < "3.11" and (sys_platform != "win32" or os_name != "nt") -pure-eval==0.2.2 ; python_version >= "3.8" and python_version < "3.11" -pyarrow==13.0.0 ; python_version >= "3.8" and python_version < "3.11" -pycifrw==4.4.6 ; python_version >= "3.8" and python_version < "3.11" -pycparser==2.21 ; python_version >= "3.8" and python_version < "3.11" -pyerfa==2.0.0.3 ; python_version >= "3.8" and python_version < "3.11" -pyfai==2023.9.0 ; python_version >= "3.8" and python_version < "3.11" -pygments==2.16.1 ; python_version >= "3.8" and python_version < "3.11" -pynxtools==0.0.5 ; python_version >= "3.8" and python_version < "3.11" -pyparsing==3.1.1 ; python_version >= "3.8" and python_version < "3.11" -python-dateutil==2.8.2 ; python_version >= "3.8" and python_version < "3.11" -python-json-logger==2.0.7 ; python_version >= "3.8" and python_version < "3.11" -pytz-deprecation-shim==0.1.0.post0 ; python_version >= "3.8" and python_version < "3.11" -pytz==2023.3.post1 ; python_version >= "3.8" and python_version < "3.11" -pywavelets==1.4.1 ; python_version >= "3.8" and python_version < "3.11" -pywin32==306 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.8" and python_version < "3.11" -pywinpty==2.0.12 ; python_version >= "3.8" and python_version < "3.11" and os_name == "nt" -pyxem==0.15.1 ; python_version >= "3.8" and python_version < "3.11" -pyyaml==6.0.1 ; python_version >= "3.8" and python_version < "3.11" -pyzmq==25.1.1 ; python_version >= "3.8" and python_version < "3.11" -qtconsole==5.4.4 ; python_version >= "3.8" and python_version < "3.11" -qtpy==2.4.1 ; python_version >= "3.8" and python_version < "3.11" -radioactivedecay==0.4.21 ; python_version >= "3.8" and python_version < "3.11" -recommonmark==0.7.1 ; python_version >= "3.8" and python_version < "3.11" -referencing==0.30.2 ; python_version >= "3.8" and python_version < "3.11" -requests-cache==1.1.0 ; python_version >= "3.8" and python_version < "3.11" -requests==2.31.0 ; python_version >= "3.8" and python_version < "3.11" -rfc3339-validator==0.1.4 ; python_version >= "3.8" and python_version < "3.11" -rfc3986-validator==0.1.1 ; python_version >= "3.8" and python_version < "3.11" -rpds-py==0.12.0 ; python_version >= "3.8" and python_version < "3.11" -scikit-image==0.21.0 ; python_version >= "3.8" and python_version < "3.11" -scikit-learn==1.3.2 ; python_version >= "3.8" and python_version < "3.11" -scipy==1.10.1 ; python_version >= "3.8" and python_version < "3.11" -send2trash==1.8.2 ; python_version >= "3.8" and python_version < "3.11" -setuptools-scm==8.0.4 ; python_version >= "3.8" and python_version < "3.11" -setuptools==68.2.2 ; python_version >= "3.8" and python_version < "3.11" -silx==1.1.2 ; python_version >= "3.8" and python_version < "3.11" -six==1.16.0 ; python_version >= "3.8" and python_version < "3.11" -smmap==5.0.1 ; python_version >= "3.8" and python_version < "3.11" -sniffio==1.3.0 ; python_version >= "3.8" and python_version < "3.11" -snowballstemmer==2.2.0 ; python_version >= "3.8" and python_version < "3.11" -soupsieve==2.5 ; python_version >= "3.8" and python_version < "3.11" -sparse==0.14.0 ; python_version >= "3.8" and python_version < "3.11" -sphinx-autodoc-typehints[docs]==1.24.1 ; python_version >= "3.8" and python_version < "3.11" -sphinx-basic-ng==1.0.0b2 ; python_version >= "3.8" and python_version < "3.11" -sphinx-rtd-theme==1.3.0 ; python_version >= "3.8" and python_version < "3.11" -sphinx-rtd-theme[docs]==1.3.0 ; python_version >= "3.8" and python_version < "3.11" -sphinx==7.1.2 ; python_version >= "3.8" and python_version < "3.11" -sphinx[docs]==7.1.2 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-applehelp==1.0.4 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-devhelp==1.0.2 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-htmlhelp==2.0.1 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-jquery==4.1 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-jsmath==1.0.1 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-qthelp==1.0.3 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-serializinghtml==1.1.5 ; python_version >= "3.8" and python_version < "3.11" -sphinxcontrib-websupport==1.2.4 ; python_version >= "3.8" and python_version < "3.11" -stack-data==0.6.3 ; python_version >= "3.8" and python_version < "3.11" -symmetrize==0.5.5 ; python_version >= "3.8" and python_version < "3.11" -sympy==1.12 ; python_version >= "3.8" and python_version < "3.11" -terminado==0.17.1 ; python_version >= "3.8" and python_version < "3.11" -threadpoolctl==3.2.0 ; python_version >= "3.8" and python_version < "3.11" -tifffile==2022.10.10 ; python_version >= "3.8" and python_version < "3.11" -tinycss2==1.2.1 ; python_version >= "3.8" and python_version < "3.11" -tomli==2.0.1 ; python_version >= "3.8" and python_version < "3.11" -tomlkit[docs]==0.12.2 ; python_version >= "3.8" and python_version < "3.11" -toolz==0.12.0 ; python_version >= "3.8" and python_version < "3.11" -tornado==6.3.3 ; python_version >= "3.8" and python_version < "3.11" -tqdm==4.66.1 ; python_version >= "3.8" and python_version < "3.11" -traitlets==5.13.0 ; python_version >= "3.8" and python_version < "3.11" -traits==6.4.3 ; python_version >= "3.8" and python_version < "3.11" -transforms3d==0.4.1 ; python_version >= "3.8" and python_version < "3.11" -types-python-dateutil==2.8.19.14 ; python_version >= "3.8" and python_version < "3.11" -typing-extensions==4.8.0 ; python_version >= "3.8" and python_version < "3.11" -tzdata==2023.3 ; python_version >= "3.8" and python_version < "3.11" -tzlocal==4.3 ; python_version >= "3.8" and python_version < "3.11" -uncertainties==3.1.7 ; python_version >= "3.8" and python_version < "3.11" -uri-template==1.3.0 ; python_version >= "3.8" and python_version < "3.11" -url-normalize==1.4.3 ; python_version >= "3.8" and python_version < "3.11" -urllib3==2.0.7 ; python_version >= "3.8" and python_version < "3.11" -wcwidth==0.2.9 ; python_version >= "3.8" and python_version < "3.11" -webcolors==1.13 ; python_version >= "3.8" and python_version < "3.11" -webencodings==0.5.1 ; python_version >= "3.8" and python_version < "3.11" -websocket-client==1.6.4 ; python_version >= "3.8" and python_version < "3.11" -widgetsnbextension==4.0.9 ; python_version >= "3.8" and python_version < "3.11" -xarray==0.20.2 ; python_version >= "3.8" and python_version < "3.11" -zarr==2.16.1 ; python_version >= "3.8" and python_version < "3.11" and (platform_machine == "x86_64" or platform_machine == "AMD64") -zipfile37==0.1.3 ; python_version >= "3.8" and python_version < "3.11" -zipp==3.17.0 ; python_version >= "3.8" and python_version < "3.11" diff --git a/docs/sed/loader.rst b/docs/sed/loader.rst index 59cc26ed..a289cd1b 100644 --- a/docs/sed/loader.rst +++ b/docs/sed/loader.rst @@ -31,6 +31,14 @@ FlashLoader :members: :undoc-members: +.. automodule:: sed.loader.flash.metadata + :members: + :undoc-members: + +.. automodule:: sed.loader.flash.utils + :members: + :undoc-members: + Utilities ################################################### .. automodule:: sed.loader.utils diff --git a/poetry.lock b/poetry.lock index 10b1b817..8326181b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1399,6 +1399,29 @@ gitdb = ">=4.0.1,<5" [package.extras] test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest", "pytest-cov", "pytest-instafail", "pytest-subtests", "pytest-sugar"] +[[package]] +name = "h5grove" +version = "1.3.0" +description = "Core utilities to serve HDF5 file contents" +optional = true +python-versions = ">=3.6" +files = [ + {file = "h5grove-1.3.0-py3-none-any.whl", hash = "sha256:340d7b47e9957bd666b92712d62d0841797449774a738ec8f519ed80ba9a13fb"}, + {file = "h5grove-1.3.0.tar.gz", hash = "sha256:e8f052ff497f0ff42477a24511bbf0d8a1cf0b6e7aea31957bdec7b4baae2c9a"}, +] + +[package.dependencies] +h5py = ">=2.9" +numpy = "*" +orjson = "*" +tifffile = "*" + +[package.extras] +dev = ["black", "bump2version", "check-manifest", "flake8", "h5grove[fastapi]", "h5grove[flask]", "h5grove[tornado]", "httpx (>=0.23)", "invoke", "mypy", "myst-parser", "pytest", "pytest-benchmark", "pytest-cov", "pytest-tornado", "sphinx", "sphinx-argparse", "types-contextvars", "types-dataclasses", "types-orjson", "types-pkg-resources"] +fastapi = ["fastapi", "uvicorn"] +flask = ["Flask", "Flask-Compress", "Flask-Cors"] +tornado = ["tornado"] + [[package]] name = "h5py" version = "3.10.0" @@ -2169,6 +2192,25 @@ docs = ["jsx-lexer", "myst-parser", "pydata-sphinx-theme (>=0.13.0)", "pytest", docs-screenshots = ["altair (==5.0.1)", "ipython (==8.14.0)", "ipywidgets (==8.0.6)", "jupyterlab-geojson (==3.4.0)", "jupyterlab-language-pack-zh-cn (==4.0.post0)", "matplotlib (==3.7.1)", "nbconvert (>=7.0.0)", "pandas (==2.0.2)", "scipy (==1.10.1)", "vega-datasets (==0.9.0)"] test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter (>=0.5.3)", "pytest-timeout", "pytest-tornasync", "requests", "requests-cache", "virtualenv"] +[[package]] +name = "jupyterlab-h5web" +version = "11.0.0" +description = "A JupyterLab extension to explore and visualize HDF5 file contents." +optional = true +python-versions = ">=3.8" +files = [ + {file = "jupyterlab_h5web-11.0.0-py3-none-any.whl", hash = "sha256:39d5ef3dcb9f2fdca1abd33218185c603c60bf8629090a5b3cdc98a0605cae14"}, + {file = "jupyterlab_h5web-11.0.0.tar.gz", hash = "sha256:6bc3f9b344a3509e692ebc9e6044c53f3c6083d988a5d0371b6f4525e557c53e"}, +] + +[package.dependencies] +h5grove = "1.3.0" +h5py = ">=3.5" +jupyter-server = ">=1.6,<3" + +[package.extras] +full = ["hdf5plugin"] + [[package]] name = "jupyterlab-pygments" version = "0.2.2" @@ -2968,6 +3010,25 @@ traitlets = ">=5.1" docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] test = ["pep440", "pre-commit", "pytest", "testpath"] +[[package]] +name = "nbsphinx" +version = "0.9.3" +description = "Jupyter Notebook Tools for Sphinx" +optional = true +python-versions = ">=3.6" +files = [ + {file = "nbsphinx-0.9.3-py3-none-any.whl", hash = "sha256:6e805e9627f4a358bd5720d5cbf8bf48853989c79af557afd91a5f22e163029f"}, + {file = "nbsphinx-0.9.3.tar.gz", hash = "sha256:ec339c8691b688f8676104a367a4b8cf3ea01fd089dc28d24dec22d563b11562"}, +] + +[package.dependencies] +docutils = "*" +jinja2 = "*" +nbconvert = "!=5.4" +nbformat = "*" +sphinx = ">=1.8" +traitlets = ">=5" + [[package]] name = "nest-asyncio" version = "1.5.8" @@ -3389,6 +3450,65 @@ dev = ["black[jupyter]", "coverage (>=5.0)", "ipykernel", "isort (>=5.10)", "man doc = ["ipykernel", "memory-profiler", "nbsphinx (>=0.7)", "numpydoc", "pydata-sphinx-theme (>=0.13.1)", "scikit-image", "scikit-learn", "sphinx (>=3.0.2)", "sphinx-codeautolink[ipython]", "sphinx-copybutton (>=0.2.5)", "sphinx-design", "sphinx-gallery (<0.11)", "sphinx-last-updated-by-git", "sphinxcontrib-bibtex (>=1.0)"] tests = ["coverage (>=5.0)", "numpydoc", "pytest (>=5.4)", "pytest-cov (>=2.8.1)", "pytest-xdist"] +[[package]] +name = "orjson" +version = "3.9.10" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = true +python-versions = ">=3.8" +files = [ + {file = "orjson-3.9.10-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9"}, + {file = "orjson-3.9.10-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83"}, + {file = "orjson-3.9.10-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d"}, + {file = "orjson-3.9.10-cp310-none-win32.whl", hash = "sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1"}, + {file = "orjson-3.9.10-cp310-none-win_amd64.whl", hash = "sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7"}, + {file = "orjson-3.9.10-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb"}, + {file = "orjson-3.9.10-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499"}, + {file = "orjson-3.9.10-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3"}, + {file = "orjson-3.9.10-cp311-none-win32.whl", hash = "sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8"}, + {file = "orjson-3.9.10-cp311-none-win_amd64.whl", hash = "sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616"}, + {file = "orjson-3.9.10-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d"}, + {file = "orjson-3.9.10-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921"}, + {file = "orjson-3.9.10-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca"}, + {file = "orjson-3.9.10-cp312-none-win_amd64.whl", hash = "sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d"}, + {file = "orjson-3.9.10-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b"}, + {file = "orjson-3.9.10-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777"}, + {file = "orjson-3.9.10-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8"}, + {file = "orjson-3.9.10-cp38-none-win32.whl", hash = "sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643"}, + {file = "orjson-3.9.10-cp38-none-win_amd64.whl", hash = "sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5"}, + {file = "orjson-3.9.10-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521"}, + {file = "orjson-3.9.10-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864"}, + {file = "orjson-3.9.10-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade"}, + {file = "orjson-3.9.10-cp39-none-win32.whl", hash = "sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088"}, + {file = "orjson-3.9.10-cp39-none-win_amd64.whl", hash = "sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff"}, + {file = "orjson-3.9.10.tar.gz", hash = "sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1"}, +] + [[package]] name = "overrides" version = "7.4.0" @@ -5931,10 +6051,10 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [extras] -docs = ["sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "tomlkit"] -notebook = ["ipykernel", "jupyter"] +docs = ["nbsphinx", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "tomlkit"] +notebook = ["ipykernel", "jupyter", "jupyterlab-h5web"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "74ca033105384efd95c12cbd728e1bc570c2d5d147a5d6a5effc9f7f279bbc62" +content-hash = "10a18aa12d733655eb7581612b98c5c0ed22191ddb3bfe7f0a4f8e66b0e5e6e4" diff --git a/pyproject.toml b/pyproject.toml index d5f05fcf..d9bda125 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,15 +33,17 @@ joblib = "^1.2.0" pyarrow = "^13.0.0" jupyter = {version = "^1.0.0", extras = ["notebook"], optional = true} ipykernel = {version = "^6.9.1", extras = ["notebook"], optional = true} +jupyterlab-h5web = {version = ">=7.0.0", extras = ["notebook"], optional = true} sphinx = {version = ">4.4.0", extras = ["docs"], optional = true} sphinx-rtd-theme = {version = ">1.0.0", extras = ["docs"], optional = true} tomlkit = {version = ">0.10.0", extras = ["docs"], optional = true} sphinx-autodoc-typehints = {version = ">1.17.0", extras = ["docs"], optional = true} +nbsphinx = {version = "^0.9.3", extras = ["docs"], optional = true} [tool.poetry.extras] -notebook = ["jupyter", "ipykernel"] -docs = ["Sphinx", "sphinx-rtd-theme", "tomlkit", "sphinx-autodoc-typehints"] +notebook = ["jupyter", "ipykernel", "jupyterlab-h5web"] +docs = ["Sphinx", "sphinx-rtd-theme", "tomlkit", "sphinx-autodoc-typehints", "nbsphinx"] [tool.poetry.group.dev.dependencies] pytest = "^7.0.1" diff --git a/tutorial/1 - Binning fake data.ipynb b/tutorial/1_binning_fake_data.ipynb similarity index 94% rename from tutorial/1 - Binning fake data.ipynb rename to tutorial/1_binning_fake_data.ipynb index 6b4ab0b3..3df656c9 100644 --- a/tutorial/1 - Binning fake data.ipynb +++ b/tutorial/1_binning_fake_data.ipynb @@ -7,7 +7,7 @@ "tags": [] }, "source": [ - "# Binning demonstration on locally generated fake data\n", + "### Binning demonstration on locally generated fake data\n", "In this example, we generate a table with random data simulating a single event dataset.\n", "We showcase the binning method, first on a simple single table using the bin_partition method and then in the distributed mehthod bin_dataframe, using daks dataframes.\n", "The first method is never really called directly, as it is simply the function called by the bin_dataframe on each partition of the dask dataframe." @@ -29,7 +29,6 @@ "\n", "import matplotlib.pyplot as plt\n", "\n", - "\n", "sys.path.append(\"../\")\n", "from sed.binning import bin_partition, bin_dataframe" ] @@ -39,7 +38,7 @@ "id": "42a6afaa-17dd-4637-ba75-a28c4ead1adf", "metadata": {}, "source": [ - "# Generate Fake Data" + "## Generate Fake Data" ] }, { @@ -60,7 +59,7 @@ "id": "6902fd56-1456-4da6-83a4-0f3f6b831eb6", "metadata": {}, "source": [ - "# Define the binning range" + "## Define the binning range" ] }, { @@ -81,7 +80,7 @@ "id": "00054b5d-fc96-4959-b562-7cb8545a9535", "metadata": {}, "source": [ - "# Compute the binning along the pandas dataframe" + "## Compute the binning along the pandas dataframe" ] }, { @@ -118,7 +117,7 @@ "id": "e632dc1d-5eb5-4621-8bef-4438ce2c6a0c", "metadata": {}, "source": [ - "# Transform to dask dataframe" + "## Transform to dask dataframe" ] }, { @@ -137,7 +136,7 @@ "id": "01066d40-010a-490b-9033-7339e5a21b26", "metadata": {}, "source": [ - "# compute distributed binning on the partitioned dask dataframe\n", + "## compute distributed binning on the partitioned dask dataframe\n", "In this example, the small dataset does not give significant improvement over the pandas implementation, at least using this number of partitions.\n", "A single partition would be faster (you can try...) but we use multiple for demonstration purpouses." ] diff --git a/tutorial/3 - Conversion Pipeline for example time-resolved ARPES data.ipynb b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb similarity index 93% rename from tutorial/3 - Conversion Pipeline for example time-resolved ARPES data.ipynb rename to tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb index 1ae9785c..b920bd20 100644 --- a/tutorial/3 - Conversion Pipeline for example time-resolved ARPES data.ipynb +++ b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb @@ -8,7 +8,7 @@ "tags": [] }, "source": [ - "# Demonstration of the conversion pipeline using time-resolved ARPES data stored on Zenode\n", + "# Demonstration of the conversion pipeline using time-resolved ARPES data stored on Zenodo\n", "In this example, we pull some time-resolved ARPES data from Zenodo, and load it into the sed package using functions of the mpes package. Then, we run a conversion pipeline on it, containing steps for visualizing the channels, correcting image distortions, calibrating the momentum space, correcting for energy distortions and calibrating the energy axis. Finally, the data are binned in calibrated axes.\n", "For performance reasons, best store the data on a locally attached storage (no network drive). This can also be achieved transparently using the included MirrorUtil class." ] @@ -37,7 +37,7 @@ "id": "42a6afaa-17dd-4637-ba75-a28c4ead1adf", "metadata": {}, "source": [ - "# Load Data" + "## Load Data" ] }, { @@ -47,10 +47,10 @@ "metadata": {}, "outputs": [], "source": [ - "data_path = '../../' # Put in Path to a storage of at least 20 Gbyte free space.\n", + "data_path = '.' # Put in Path to a storage of at least 20 Gbyte free space.\n", "if not os.path.exists(data_path + \"/WSe2.zip\"):\n", - " os.system(f\"curl --output {data_path}/WSe2.zip https://zenodo.org/record/6369728/files/WSe2.zip\")\n", - "if not os.path.isdir(data_path + \"/Scan049_1\") or not os.path.isdir(data_path + \"energycal_2019_01_08/\"):\n", + " os.system(f\"curl -L --output {data_path}/WSe2.zip https://zenodo.org/record/6369728/files/WSe2.zip\")\n", + "if not os.path.isdir(data_path + \"/Scan049_1\") or not os.path.isdir(data_path + \"/energycal_2019_01_08/\"):\n", " os.system(f\"unzip -d {data_path} -o {data_path}/WSe2.zip\")" ] }, @@ -97,7 +97,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Apply jittering to X, Y, t, ADC columns. \n", + "# Apply jittering to X, Y, t, ADC columns.\n", "# Columns are defined in the config, or can be provided as list.\n", "sp.add_jitter()" ] @@ -123,8 +123,9 @@ "id": "70aa4343", "metadata": {}, "source": [ - "# Distortion correction and Momentum Calibration workflow\n", - "### 1. step: \n", + "## Distortion correction and Momentum Calibration workflow\n", + "### Distortion correction\n", + "#### 1. step: \n", "Bin and load part of the dataframe in detector coordinates, and choose energy plane where high-symmetry points can well be identified. Either use the interactive tool, or pre-select the range:" ] }, @@ -145,7 +146,7 @@ "id": "fee3ca76", "metadata": {}, "source": [ - "### 2. Step:\n", + "#### 2. Step:\n", "Next, we select a number of features corresponding to the rotational symmetry of the material, plus the center. These can either be auto-detected (for well-isolated points), or provided as a list (these can be read-off the graph in the cell above).\n", "These are then symmetrized according to the rotational symmetry, and a spline-warping correction for the x/y coordinates is calculated, which corrects for any geometric distortions from the perfect n-fold rotational symmetry." ] @@ -162,8 +163,8 @@ "# Manual selection: Use a GUI tool to select peaks:\n", "#sp.define_features(rotation_symmetry=6, include_center=True)\n", "#sp.generate_splinewarp(rotation_symmetry=6, include_center=True, fwhm=10, sigma=12, sigma_radius=4)\n", - "# Autodetect: Uses the DAOStarFinder routine to locate maxima. \n", - "# Parameters are: \n", + "# Autodetect: Uses the DAOStarFinder routine to locate maxima.\n", + "# Parameters are:\n", "# fwhm: Full-width at half maximum of peaks.\n", "# sigma: Number of standard deviations above the mean value of the image peaks must have.\n", "# sigma_radius: number of standard deviations around a peak that peaks are fitted\n", @@ -176,7 +177,7 @@ "id": "f7519ff8", "metadata": {}, "source": [ - "### 3. Step: \n", + "#### 3. Step: \n", "Generate nonlinear correction using splinewarp algorithm. If no landmarks have been defined in previous step, default parameters from the config are used" ] }, @@ -217,7 +218,7 @@ "id": "b5e69ffa", "metadata": {}, "source": [ - "### 4. Step:\n", + "#### 4. Step:\n", "To adjust scaling, position and orientation of the corrected momentum space image, you can apply further affine transformations to the distortion correction field. Here, first a postential scaling is applied, next a translation, and finally a rotation around the center of the image (defined via the config). One can either use an interactive tool, or provide the adjusted values and apply them directly." ] }, @@ -238,7 +239,7 @@ "id": "a78a68e9", "metadata": {}, "source": [ - "### 5. Step:\n", + "#### 5. Step:\n", "Finally, the momentum correction is applied to the dataframe, and corresponding meta data are stored" ] }, @@ -258,8 +259,8 @@ "id": "d9810488", "metadata": {}, "source": [ - "## Momentum calibration workflow\n", - "### 1. Step:\n", + "### Momentum calibration workflow\n", + "#### 1. Step:\n", "First, the momentum scaling needs to be calibtrated. Either, one can provide the coordinates of one point outside the center, and provide its distane to the Brillouin zone center (which is assumed to be located in the center of the image), one can specify two points on the image and their distance (where the 2nd point marks the BZ center),or one can provide absolute k-coordinates of two distinct momentum points.\n", "\n", "If no points are provided, an interactive tool is created. Here, left mouse click selectes the off-center point (brillouin_zone_cetnered=True) or toggle-selects the off-center and center point." @@ -285,7 +286,7 @@ "id": "1a3697b1", "metadata": {}, "source": [ - "#### Optional (Step 1a): \n", + "##### Optional (Step 1a): \n", "Save momentum calibration parameters to configuration file in current data folder: " ] }, @@ -306,7 +307,7 @@ "id": "c2f8a513", "metadata": {}, "source": [ - "### 2. Step:\n", + "#### 2. Step:\n", "Now, the distortion correction and momentum calibration needs to be applied to the dataframe." ] }, @@ -326,7 +327,7 @@ "id": "0bce2388", "metadata": {}, "source": [ - "# Energy Correction (optional)\n", + "## Energy Correction (optional)\n", "The purpose of the energy correction is to correct for any momentum-dependent distortion of the energy axis, e.g. from geometric effects in the flight tube, or from space charge" ] }, @@ -336,7 +337,7 @@ "id": "5289de59", "metadata": {}, "source": [ - "### 1st step:\n", + "#### 1st step:\n", "Here, one can select the functional form to be used, and adjust its parameters. The binned data used for the momentum calibration is plotted around the Fermi energy (defined by tof_fermi), and the correction function is plotted ontop. Possible correction functions are: \"sperical\" (parameter: diameter), \"Lorentzian\" (parameter: gamma), \"Gaussian\" (parameter: sigma), and \"Lorentzian_asymmetric\" (parameters: gamma, amplitude2, gamma2).\n", "\n", "One can either use an interactive alignment tool, or provide parameters directly." @@ -358,7 +359,7 @@ "id": "e43fbf33", "metadata": {}, "source": [ - "#### Optional (Step 1a): \n", + "##### Optional (Step 1a): \n", "Save energy correction parameters to configuration file in current data folder: " ] }, @@ -379,7 +380,7 @@ "id": "41a6a3e6", "metadata": {}, "source": [ - "### 2. Step\n", + "#### 2. Step\n", "After adjustment, the energy correction is directly applied to the TOF axis." ] }, @@ -399,7 +400,7 @@ "id": "8b571b4c", "metadata": {}, "source": [ - "# 3. Energy calibration\n", + "## 3. Energy calibration\n", "For calibrating the energy axis, a set of data taken at different bias voltages around the value where the measurement was taken is required." ] }, @@ -409,7 +410,7 @@ "id": "6bc28642", "metadata": {}, "source": [ - "### 1. Step:\n", + "#### 1. Step:\n", "In a first step, the data are loaded, binned along the TOF dimension, and normalized. The used bias voltages can be either provided, or read from attributes in the source files if present." ] }, @@ -421,11 +422,11 @@ "outputs": [], "source": [ "# Load energy calibration EDCs\n", - "energycalfolder = data_path + \"energycal_2019_01_08/\"\n", + "energycalfolder = data_path + \"/energycal_2019_01_08/\"\n", "scans = np.arange(1,12)\n", "voltages = np.arange(12,23,1)\n", "files = [energycalfolder + r'Scan' + str(num).zfill(3) + '_' + str(num+11) + '.h5' for num in scans]\n", - "sp.load_bias_series(files, normalize=True, biases=voltages, ranges=[(64000, 75000)])" + "sp.load_bias_series(data_files=files, normalize=True, biases=voltages, ranges=[(64000, 75000)])" ] }, { @@ -434,7 +435,7 @@ "id": "314a79c8", "metadata": {}, "source": [ - "### 2. Step:\n", + "#### 2. Step:\n", "Next, the same peak or feature needs to be selected in each curve. For this, one needs to define \"ranges\" for each curve, within which the peak of interest is located. One can either provide these ranges manually, or provide one range for a \"reference\" curve, and infer the ranges for the other curves using a dynamic time warping algorithm." ] }, @@ -450,7 +451,7 @@ "# sp.find_bias_peaks(ranges=rg, infer_others=False)\n", "# Option 2 = specify the range for one curve and infer the others\n", "# This will open an interactive tool to select the correct ranges for the curves.\n", - "# IMPORTANT: Don't choose the range too narrow about a peak, and choose a refid \n", + "# IMPORTANT: Don't choose the range too narrow about a peak, and choose a refid\n", "# somewhere in the middle or towards larger biases!\n", "rg = (66100, 67000)\n", "sp.find_bias_peaks(ranges=rg, ref_id=5, infer_others=True, apply=True)" @@ -462,7 +463,7 @@ "id": "b2638818", "metadata": {}, "source": [ - "### 3. Step:\n", + "#### 3. Step:\n", "Next, the detected peak positions and bias voltages are used to determine the calibration function. This can be either done by fitting the functional form d^2/(t-t0)^2 via lmfit (\"lmfit\"), or using a polynomial approxiamtion (\"lstsq\" or \"lsqr\"). Here, one can also define a reference id, and a reference energy. Those define the absolute energy position of the feature used for calibration in the \"reference\" trace, at the bias voltage where the final measurement has been performed. The energy scale can be either \"kientic\" (decreasing energy with increasing TOF), or \"binding\" (increasing energy with increasing TOF).\n", "\n", "After calculating the calibration, all traces corrected with the calibration are plotted ontop of each other, the calibration function together with the extracted features is plotted." @@ -488,7 +489,7 @@ "id": "df63c6c7", "metadata": {}, "source": [ - "#### Optional (Step 3a): \n", + "##### Optional (Step 3a): \n", "Save energy calibration parameters to configuration file in current data folder: " ] }, @@ -509,7 +510,7 @@ "id": "563709c7", "metadata": {}, "source": [ - "### 4. Step:\n", + "#### 4. Step:\n", "Finally, the the energy axis is added to the dataframe." ] }, @@ -529,7 +530,7 @@ "id": "b2d8cdf9", "metadata": {}, "source": [ - "# 4. Delay calibration:\n", + "## 4. Delay calibration:\n", "The delay axis is calculated from the ADC input column based on the provided delay range. ALternatively, the delay scan range can also be extracted from attributes inside a source file, if present." ] }, @@ -554,7 +555,7 @@ "id": "d9d0b018", "metadata": {}, "source": [ - "# 5. Visualization of calibrated histograms\n", + "## 5. Visualization of calibrated histograms\n", "With all calibrated axes present in the dataframe, we can visualize the corresponding histograms, and determine the respective binning ranges" ] }, @@ -576,7 +577,7 @@ "id": "6902fd56-1456-4da6-83a4-0f3f6b831eb6", "metadata": {}, "source": [ - "# Define the binning ranges and compute calibrated data volume" + "## Define the binning ranges and compute calibrated data volume" ] }, { @@ -598,7 +599,7 @@ "id": "523794dc", "metadata": {}, "source": [ - "# Some visualization:" + "## Some visualization:" ] }, { @@ -641,8 +642,8 @@ "interpreter": { "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243" }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "kernelspec": { + "display_name": "sed_poetry", "language": "python", "name": "python3" }, diff --git a/tutorial/4 - Metadata collection and export to NeXus.ipynb b/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb similarity index 96% rename from tutorial/4 - Metadata collection and export to NeXus.ipynb rename to tutorial/3_metadata_collection_and_export_to_NeXus.ipynb index f7e629ec..b704a36e 100644 --- a/tutorial/4 - Metadata collection and export to NeXus.ipynb +++ b/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb @@ -36,7 +36,7 @@ "id": "42a6afaa-17dd-4637-ba75-a28c4ead1adf", "metadata": {}, "source": [ - "# Load Data" + "## Load Data" ] }, { @@ -46,10 +46,10 @@ "metadata": {}, "outputs": [], "source": [ - "data_path = '../../' # Put in Path to a storage of at least 20 Gbyte free space.\n", + "data_path = '.' # Put in Path to a storage of at least 20 Gbyte free space.\n", "if not os.path.exists(data_path + \"/WSe2.zip\"):\n", - " os.system(f\"curl --output {data_path}/WSe2.zip https://zenodo.org/record/6369728/files/WSe2.zip\")\n", - "if not os.path.isdir(data_path + \"/Scan049_1\") or not os.path.isdir(data_path + \"energycal_2019_01_08/\"):\n", + " os.system(f\"curl -L --output {data_path}/WSe2.zip https://zenodo.org/record/6369728/files/WSe2.zip\")\n", + "if not os.path.isdir(data_path + \"/Scan049_1\") or not os.path.isdir(data_path + \"/energycal_2019_01_08/\"):\n", " os.system(f\"unzip -d {data_path} -o {data_path}/WSe2.zip\")" ] }, @@ -96,7 +96,7 @@ "metadata['instrument']['beam']['probe']['pulse_duration'] = 20.\n", "metadata['instrument']['beam']['probe']['frequency'] = 500.\n", "metadata['instrument']['beam']['probe']['incident_polarization'] = [1, 1, 0, 0] # p pol Stokes vector\n", - "metadata['instrument']['beam']['probe']['extent'] = [80., 80.] \n", + "metadata['instrument']['beam']['probe']['extent'] = [80., 80.]\n", "#pump beam\n", "metadata['instrument']['beam']['pump']={}\n", "metadata['instrument']['beam']['pump']['incident_energy'] = 1.55\n", @@ -104,10 +104,10 @@ "metadata['instrument']['beam']['pump']['pulse_duration'] = 35.\n", "metadata['instrument']['beam']['pump']['frequency'] = 500.\n", "metadata['instrument']['beam']['pump']['incident_polarization'] = [1, -1, 0, 0] # s pol Stokes vector\n", - "metadata['instrument']['beam']['pump']['incident_wavelength'] = 800. \n", + "metadata['instrument']['beam']['pump']['incident_wavelength'] = 800.\n", "metadata['instrument']['beam']['pump']['average_power'] = 300.\n", "metadata['instrument']['beam']['pump']['pulse_energy'] = metadata['instrument']['beam']['pump']['average_power']/metadata['instrument']['beam']['pump']['frequency']#µJ\n", - "metadata['instrument']['beam']['pump']['extent'] = [230., 265.] \n", + "metadata['instrument']['beam']['pump']['extent'] = [230., 265.]\n", "metadata['instrument']['beam']['pump']['fluence'] = 0.15\n", "\n", "#sample\n", @@ -158,7 +158,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Apply jittering to X, Y, t, ADC columns. \n", + "# Apply jittering to X, Y, t, ADC columns.\n", "sp.add_jitter()" ] }, @@ -246,7 +246,7 @@ "id": "6902fd56-1456-4da6-83a4-0f3f6b831eb6", "metadata": {}, "source": [ - "# Compute final data volume" + "## Compute final data volume" ] }, { diff --git a/tutorial/5 - hextof workflow.ipynb b/tutorial/4_hextof_workflow.ipynb similarity index 100% rename from tutorial/5 - hextof workflow.ipynb rename to tutorial/4_hextof_workflow.ipynb diff --git a/tutorial/sed_config.yaml b/tutorial/sed_config.yaml index 8565c8fb..5577b42e 100644 --- a/tutorial/sed_config.yaml +++ b/tutorial/sed_config.yaml @@ -25,18 +25,20 @@ momentum: y_center: 256.0 correction: feature_points: - - - 203.11575556771575 - - 343.1023874450215 - - - 299.9643115931048 - - 346.2942034781325 - - - 351.05271790029917 - - 244.87949469676045 - - - 305.76331680416877 - - 150.31266296600884 - - - 199.64692385066613 - - 152.8942716287488 - - - 153.0 - - 243.0 - - - 249.32627242026467 - - 249.34641745326562 + - - 202.99667164649654 + - 342.9841737181237 + - - 299.87095669185146 + - 346.1951264748602 + - - 350.95080745426304 + - 244.7908230308385 + - - 305.6268110815786 + - 150.20132111991873 + - - 199.5398499983996 + - 152.77801048162016 + - - 153.40923361300395 + - 243.06399842230255 + - - 249.232157094759 + - 249.2577242394875 + include_center: true rotation_symmetry: 6 + use_center: true