diff --git a/.github/requirements_ci.txt b/.github/requirements_ci.txt index af4d284..3f97edf 100644 --- a/.github/requirements_ci.txt +++ b/.github/requirements_ci.txt @@ -4,3 +4,4 @@ jupyterlab>=3.0.0 matplotlib>=3.3.0 yaargh>=0.28.0 +papermill>=2.3.4 diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index ea9b3f8..aee5e2a 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -49,7 +49,7 @@ jobs: # If not provided (push and pull_request event) it uses the default branch ref: ${{ github.event.inputs.pyglotaran_branch }} - name: Set up Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install pyglotaran @@ -68,21 +68,66 @@ jobs: examples_branch: ${{ github.event.inputs.pyglotaran_examples_branch }} - name: Upload Example Plots Artifact - uses: actions/upload-artifact@v3 + if: always() + uses: actions/upload-artifact@v4 with: - name: example-results - path: ${{ steps.example-run.outputs.plots-path }} + name: example-notebooks-${{ matrix.example_name }} + path: ${{ steps.example-run.outputs.notebook-path }} - name: Upload Example Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: example-results + name: example-results-${{ matrix.example_name }} path: ~/pyglotaran_examples_results + collect-artifacts: + if: always() + name: "Collect artifacts and reupload as bundel" + runs-on: ubuntu-latest + needs: [run-examples] + steps: + - name: Download Notebooks Artifacts + uses: actions/download-artifact@v4 + with: + path: example-notebooks + pattern: example-notebooks-* + merge-multiple: true + + - name: Upload Example Notebooks Artifact + uses: actions/upload-artifact@v4 + with: + name: example-notebooks + path: example-notebooks + overwrite: true + + - name: Delete Intermediate Notebooks artifacts + uses: GeekyEggo/delete-artifact@v5 + with: + name: example-notebooks-* + + - name: Download Result Artifacts + uses: actions/download-artifact@v4 + with: + path: example-results + pattern: example-results-* + merge-multiple: true + + - name: Upload Example Result Artifact + uses: actions/upload-artifact@v4 + with: + name: example-results + path: example-results + overwrite: true + + - name: Delete Intermediate Result artifacts + uses: GeekyEggo/delete-artifact@v5 + with: + name: example-results-* + compare-results: name: Compare Results runs-on: ubuntu-latest - needs: [run-examples] + needs: [collect-artifacts] steps: - name: Checkout compare results uses: actions/checkout@v4 @@ -92,7 +137,7 @@ jobs: path: comparison-results - name: Download result artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: example-results path: comparison-results-current @@ -108,7 +153,7 @@ jobs: echo "::endgroup::" - name: Set up Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 089be24..3e30744 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-ast - id: check-builtin-literals @@ -10,33 +10,46 @@ repos: - id: debug-statements - id: fix-encoding-pragma args: [--remove] - - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.4.7 hooks: - - id: pyupgrade - args: [--py38-plus] - - repo: https://github.com/psf/black - rev: 23.9.1 - hooks: - - id: black - language_version: python3 - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - - repo: https://github.com/asottile/yesqa - rev: v1.5.0 - hooks: - - id: yesqa - additional_dependencies: [flake8-docstrings] - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - args: [--max-line-length=99] + - id: ruff + name: "ruff sort notebooks" + args: + - "--fix" + - "--select=I" + - "--config=.ruff-notebooks.toml" + alias: isort-notebooks + types_or: [jupyter] + - id: ruff-format + name: "ruff format notebooks" + types_or: [jupyter] + args: + - "--config=.ruff-notebooks.toml" + alias: format-notebooks + - id: ruff + name: "ruff lint notebooks" + types_or: [jupyter] + args: + - "--config=.ruff-notebooks.toml" + alias: check-notebooks + - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell files: ".py|.rst" - args: ["--skip='*.ipynb'", "-L doas,DOAS"] + args: ["--ignore-words-list=doas"] + exclude: "\\.ipynb" + + # Notebook tools + - repo: https://github.com/kynan/nbstripout + rev: 0.7.1 + hooks: + - id: nbstripout + args: + - "--drop-empty-cells" + - "--extra-keys" + - "cell.metadata.tags metadata.vscode metadata.papermill metadata.language_info.version cell.metadata.papermill" diff --git a/.ruff-notebooks.toml b/.ruff-notebooks.toml new file mode 100644 index 0000000..dc34bb4 --- /dev/null +++ b/.ruff-notebooks.toml @@ -0,0 +1,10 @@ +#:schema https://json.schemastore.org/ruff.json + +extend = ".ruff.toml" + +[lint] +extend-ignore = ["D", "E402", "F404", "N811", "E703", "T201"] + +[lint.isort] +required-imports = [] +force-single-line = false diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..ca61c08 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,78 @@ +extend-exclude = ["venv", "docs/conf.py"] + +line-length = 99 + +# Assume Python 3.10. +target-version = "py310" + +# Enable using ruff with notebooks +extend-include = ["*.ipynb"] + +[lint] +select = [ + "E", # pycodestyle + "W", # pycodestyle + # "C", # mccabe + "F", # pyflakes + # "UP", # pyupgrade + # "D", # pydocstyle + "N", # pep8-naming + "YTT", # flake8-2020 + "BLE", # flake8-blind-except + # "FBT", # flake8-boolean-trap + # "B", # flake8-bugbear + "C4", # flake8-comprehensions + "T10", # flake8-debugger + "FA", # flake8-future-annotations + # "EM", # flake8-errmsg + "ISC", # flake8-implicit-str-concat + "INP", # flake8-no-pep420 + "PIE", # flake8-pie + "T20", # flake8-print + # "PT", # flake8-pytest-style + "RSE", # flake8-raise + "RET", # flake8-return + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + # "ARG", # flake8-unused-arguments + "PTH", # flake8-use-pathlib + # "ERA", # eradicate + "PD", # pandas-vet + "PGH", # pygrep-hooks + "NPY", # NumPy-specific + "RUF", # Ruff-specific +] + +# Not actually external but we deactivated them for now +external = ["D400", "D403", "D107", "D414"] + +ignore = [ + "D202", + "D213", + "D203", + "ISC002", + "RUF001", + "RUF002", + "RUF003", + "B008", + # Covered by formatter + "ISC001", +] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +unfixable = ["F401"] + +[lint.per-file-ignores] +"*/test_*.py" = ["ARG001", "RUF012", "N811", "T20", "PIE804"] +"*/__init__.py" = ["F401"] +"glotaran/builtin/io/netCDF/netCDF.py" = ["N999"] +"docs/*" = ["INP001"] +# Needs a full rewrite anyway +"glotaran/builtin/io/ascii/wavelength_time_explicit_file.py" = ["PTH"] + +[lint.isort] +required-imports = ["from __future__ import annotations"] +known-first-party = ["glotaran"] +force-single-line = true diff --git a/action.yml b/action.yml index f77d5cd..c8d41c8 100644 --- a/action.yml +++ b/action.yml @@ -19,12 +19,12 @@ inputs: default: "false" outputs: - plots-path: - description: "Plot result path of example run" - value: ${{ steps.example-run.outputs.plots-path }} example-list: description: "List of all possible example names to be used in a workflow matrix." value: ${{ steps.example-run.outputs.example-list }} + notebook-path: + description: "Path to evaluated notebook after running the example" + value: ${{ steps.example-run.outputs.notebook-path }} runs: using: "composite" @@ -62,17 +62,16 @@ runs: echo "::endgroup::" shell: bash - - name: Run example + - name: Run example notebooks id: example-run run: | echo "::group:: Running ${{ inputs.example_name }}" if [ '${{ inputs.set_example_list }}' = 'false' ] then - python pyglotaran-examples/scripts/run_examples.py ${{ inputs.example_name }} --headless 2>&1 - echo "plots-path=pyglotaran-examples/plot_results" >> $GITHUB_OUTPUT + python pyglotaran-examples/scripts/run_examples_notebooks.py ${{ inputs.example_name }} 2>&1 else - pip install yaargh - python pyglotaran-examples/scripts/run_examples.py set-gha-example-list-output + pip install yaargh papermill + python pyglotaran-examples/scripts/run_examples_notebooks.py set-gha-example-list-output fi echo "::endgroup::" diff --git a/pyglotaran_examples/.github/requirements_ci.txt b/pyglotaran_examples/.github/requirements_ci.txt new file mode 100644 index 0000000..3f97edf --- /dev/null +++ b/pyglotaran_examples/.github/requirements_ci.txt @@ -0,0 +1,7 @@ + +pyglotaran>=0.3.0 +jupyterlab>=3.0.0 +matplotlib>=3.3.0 + +yaargh>=0.28.0 +papermill>=2.3.4 diff --git a/pyglotaran_examples/.github/workflows/integration_test.yml b/pyglotaran_examples/.github/workflows/integration_test.yml new file mode 100644 index 0000000..9bc8edf --- /dev/null +++ b/pyglotaran_examples/.github/workflows/integration_test.yml @@ -0,0 +1,199 @@ +name: "Run Examples" + +on: + push: + tags: + - v** + pull_request: + workflow_dispatch: + inputs: + pyglotaran_branch: + description: "pyglotaran branch/tag to run the examples against" + required: true + default: "main" + pyglotaran_examples_branch: + description: "pyglotaran-examples branch/tag to use" + required: true + default: "main" + +jobs: + create-example-list: + name: Create Example List + runs-on: ubuntu-latest + outputs: + example-list: ${{ steps.create-example-list.outputs.example-list }} + steps: + - name: Cloning pyglotaran-examples + if: ${{ github.event.inputs.pyglotaran_examples_branch }} == "" + uses: actions/checkout@v4 + with: + path: pyglotaran-examples + - name: Set example list output + id: create-example-list + uses: ./pyglotaran-examples + with: + example_name: set example list + set_example_list: true + + run-examples: + name: "Run Example: " + runs-on: ubuntu-latest + needs: [create-example-list] + strategy: + matrix: + example_name: ${{fromJson(needs.create-example-list.outputs.example-list)}} + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + repository: "glotaran/pyglotaran" + # If not provided (push and pull_request event) it uses the main branch + ref: ${{ github.event.inputs.pyglotaran_branch != '' && github.event.inputs.pyglotaran_branch || 'main'}} + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install pyglotaran + run: | + pip install wheel + pip install . + - name: Cloning pyglotaran-examples + if: ${{ github.event.inputs.pyglotaran_examples_branch }} == "" + uses: actions/checkout@v4 + with: + path: pyglotaran-examples + - id: example-run + uses: ./pyglotaran-examples + with: + example_name: ${{ matrix.example_name }} + examples_branch: ${{ github.event.inputs.pyglotaran_examples_branch }} + + - name: Upload Example Plots Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: example-notebooks-${{ matrix.example_name }} + path: ${{ steps.example-run.outputs.notebook-path }} + + - name: Upload Example Results + uses: actions/upload-artifact@v4 + with: + name: example-results-${{ matrix.example_name }} + path: ~/pyglotaran_examples_results + + collect-artifacts: + if: always() + name: "Collect artifacts and reupload as bundel" + runs-on: ubuntu-latest + needs: [run-examples] + steps: + - name: Download Notebooks Artifacts + uses: actions/download-artifact@v4 + with: + path: example-notebooks + pattern: example-notebooks-* + merge-multiple: true + + - name: Upload Example Notebooks Artifact + uses: actions/upload-artifact@v4 + with: + name: example-notebooks + path: example-notebooks + overwrite: true + + - name: Delete Intermediate Notebooks artifacts + uses: GeekyEggo/delete-artifact@v5 + with: + name: example-notebooks-* + + - name: Download Result Artifacts + uses: actions/download-artifact@v4 + with: + path: example-results + pattern: example-results-* + merge-multiple: true + + - name: Upload Example Result Artifact + uses: actions/upload-artifact@v4 + with: + name: example-results + path: example-results + overwrite: true + + - name: Delete Intermediate Result artifacts + uses: GeekyEggo/delete-artifact@v5 + with: + name: example-results-* + + compare-results: + name: Compare Results + runs-on: ubuntu-latest + needs: [collect-artifacts] + steps: + - name: Checkout compare results + uses: actions/checkout@v4 + with: + repository: "glotaran/pyglotaran-examples" + ref: comparison-results + path: comparison-results + + - name: Download result artifact + uses: actions/download-artifact@v4 + with: + name: example-results + path: comparison-results-current + + - name: Show used versions for result creation + run: | + echo "::group:: ✔️ Compare-Results" + echo "✔️ pyglotaran-examples commit: $(< comparison-results/example_commit_sha.txt)" + echo "✔️ pyglotaran commit: $(< comparison-results/pyglotaran_commit_sha.txt)" + echo "::endgroup::" + echo "::group:: ♻️ Current-Results" + echo "♻️ pyglotaran-examples commit: $(< comparison-results-current/example_commit_sha.txt)" + echo "::endgroup::" + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Run result validator + uses: glotaran/pyglotaran-validation@main + with: + validation_name: pyglotaran-examples + + create-release: + name: "🚀 Create release assets and tag comparison-results branch" + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + runs-on: ubuntu-latest + needs: [compare-results] + steps: + - name: ⏬ Checkout compare results + uses: actions/checkout@v4 + with: + repository: "glotaran/pyglotaran-examples" + ref: comparison-results + + - name: Get tag name + id: tag + uses: devops-actions/action-get-tag@v1.0.2 + + - name: 📦 Create release asset + run: zip -r comparison-results-${{steps.tag.outputs.tag}}.zip . -x ".git/*" + + - name: 🚀⬆️ Upload Release Asset + uses: softprops/action-gh-release@v1 + with: + files: comparison-results-${{steps.tag.outputs.tag}}.zip + generate_release_notes: true + append_body: true + + - name: 📦 Create comparison-results tag + run: | + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git config user.name 'github-actions[bot]' + NEW_VERSION="comparison-results-${{steps.tag.outputs.tag}}" + git tag -a $NEW_VERSION -m "Comparison results used with release ${{steps.tag.outputs.tag}}" + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} + git push origin $NEW_VERSION diff --git a/pyglotaran_examples/__init__.py b/pyglotaran_examples/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyglotaran_examples/ex_doas_beta/ex_doas_beta.ipynb b/pyglotaran_examples/ex_doas_beta/ex_doas_beta.ipynb new file mode 100644 index 0000000..6c201f3 --- /dev/null +++ b/pyglotaran_examples/ex_doas_beta/ex_doas_beta.ipynb @@ -0,0 +1,142 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_parameters\n", + "\n", + "parameters = load_parameters(\"models/parameters.yml\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "dataset = load_dataset(\"data/2008Polli_betacar_chex_sim.nc\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"models/model.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model=model,\n", + " parameters=parameters,\n", + " data={\"dataset1\": dataset},\n", + " optimization_method=\"TrustRegionReflection\",\n", + " # maximum_number_function_evaluations=3,\n", + " maximum_number_function_evaluations=1,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "results_folder = results_folder / \"target_analysis\"\n", + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "result_plot, _ = plot_overview(result.data[\"dataset1\"], linlog=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "result.data[\"dataset1\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_doas\n", + "\n", + "plot_doas(result.data[\"dataset1\"])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/ex_doas_beta/ex_doas_beta.py b/pyglotaran_examples/ex_doas_beta/ex_doas_beta.py deleted file mode 100644 index d88aece..0000000 --- a/pyglotaran_examples/ex_doas_beta/ex_doas_beta.py +++ /dev/null @@ -1,79 +0,0 @@ -# To use this script as a 'notebook' in VS Code add '# %%' - -# %% Imports -from datetime import datetime -from pathlib import Path - -import matplotlib.pyplot as plt -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -DATA_PATH = "data/2008Polli_betacar_chex_sim.nc" -MODEL_PATH = "models/model.yml" -PARAMETERS_FILE_PATH = "models/parameters.yml" - - -# %% Define function -def run_doas_model(show_plot=False, block_plot=False): - results_folder, script_folder = setup_case_study( - output_folder_name="pyglotaran_examples_results" - ) - results_folder = results_folder / "target_analysis" - print(f"- Using folder {results_folder} to read/write files for this run") - - dataset = load_dataset(script_folder.joinpath(DATA_PATH)) - model = load_model(script_folder.joinpath(MODEL_PATH)) - parameters = load_parameters(script_folder.joinpath(PARAMETERS_FILE_PATH)) - - print(f"\n{'#'*10} DOAS Model {'#'*10}\n") - print(model.validate(parameters=parameters)) - assert model.valid(parameters=parameters) - - scheme = Scheme( - model=model, - parameters=parameters, - data={"dataset1": dataset}, - optimization_method="TrustRegionReflection", - # maximum_number_function_evaluations=3, - maximum_number_function_evaluations=1, - ) - result = optimize(scheme) - - print(f"\n{'#'*3} DOAS Model - Optimization Result {'#'*3}\n") - print(result) - print(f"\n{'#'*3} DOAS Model - Optimized Parameters {'#'*3}\n") - print(result.optimized_parameters) - - save_result(result, results_folder / "result.yml", allow_overwrite=True) - - plot_style = PlotStyle() - plt.rc("axes", prop_cycle=plot_style.cycler) - - if show_plot: - plot_style = PlotStyle() - plt.rc("axes", prop_cycle=plot_style.cycler) - - fig, _ = plot_overview(result.data["dataset1"], linlog=True, figure_only=False) - plt.rcParams["figure.figsize"] = (21, 14) - - timestamp = datetime.now().strftime("%y%m%d_%H%M") - fig.savefig(results_folder.joinpath(f"plot_overview_{timestamp}.pdf"), bbox_inches="tight") - - plt.show(block=block_plot) - script_dir = Path(__file__).resolve().parent - print(f"Script folder: {script_dir}") - script_dir.cwd() - - -# %% Main -if __name__ == "__main__": - run_doas_model(show_plot=True, block_plot=True) -else: - run_doas_model(show_plot=True, block_plot=False) diff --git a/pyglotaran_examples/ex_doas_beta/ex_doas_beta_ascii.py b/pyglotaran_examples/ex_doas_beta/ex_doas_beta_ascii.py deleted file mode 100644 index 9e9be8f..0000000 --- a/pyglotaran_examples/ex_doas_beta/ex_doas_beta_ascii.py +++ /dev/null @@ -1,78 +0,0 @@ -# To use this script as a 'notebook' in VS Code add '# %%' - -# %% Imports -from datetime import datetime -from pathlib import Path - -import matplotlib.pyplot as plt -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -DATA_PATH = "data/2008Polli_betacar_chex_sim.ascii" -MODEL_PATH = "models/model.yml" -PARAMETERS_FILE_PATH = "models/parameters.yml" - - -# %% Define function -def run_doas_model(show_plot=False, block_plot=False): - results_folder, script_folder = setup_case_study( - output_folder_name="pyglotaran_examples_results" - ) - results_folder = results_folder / "target_analysis" - print(f"- Using folder {results_folder} to read/write files for this run") - - dataset = load_dataset(script_folder.joinpath(DATA_PATH)) - model = load_model(script_folder.joinpath(MODEL_PATH)) - parameters = load_parameters(script_folder.joinpath(PARAMETERS_FILE_PATH)) - - print(f"\n{'#'*10} DOAS Model {'#'*10}\n") - print(model.validate(parameters=parameters)) - - scheme = Scheme( - model=model, - parameters=parameters, - data={"dataset1": dataset}, - optimization_method="TrustRegionReflection", - # maximum_number_function_evaluations=3, - maximum_number_function_evaluations=7, - ) - result = optimize(scheme) - - print(f"\n{'#'*3} DOAS Model - Optimization Result {'#'*3}\n") - print(result) - print(f"\n{'#'*3} DOAS Model - Optimized Parameters {'#'*3}\n") - print(result.optimized_parameters) - - save_result(result, results_folder / "result.yml", allow_overwrite=True) - - plot_style = PlotStyle() - plt.rc("axes", prop_cycle=plot_style.cycler) - - if show_plot: - plot_style = PlotStyle() - plt.rc("axes", prop_cycle=plot_style.cycler) - - fig, _ = plot_overview(result.data["dataset1"], linlog=True, figure_only=False) - plt.rcParams["figure.figsize"] = (21, 14) - - timestamp = datetime.now().strftime("%y%m%d_%H%M") - fig.savefig(results_folder.joinpath(f"plot_overview_{timestamp}.pdf"), bbox_inches="tight") - - plt.show(block=block_plot) - script_dir = Path(__file__).resolve().parent - print(f"Script folder: {script_dir}") - script_dir.cwd() - - -# %% Main -if __name__ == "__main__": - run_doas_model(show_plot=True, block_plot=True) -else: - run_doas_model(show_plot=True, block_plot=False) diff --git a/pyglotaran_examples/ex_spectral_constraints/__init__.py b/pyglotaran_examples/ex_spectral_constraints/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyglotaran_examples/ex_spectral_constraints/ex_spectral_constraints.ipynb b/pyglotaran_examples/ex_spectral_constraints/ex_spectral_constraints.ipynb new file mode 100644 index 0000000..68dc966 --- /dev/null +++ b/pyglotaran_examples/ex_spectral_constraints/ex_spectral_constraints.ipynb @@ -0,0 +1,224 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "dataset = load_dataset(\"data/data.ascii\")" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Analysis without equal area penalties\n", + "\n", + "Example of a two step optimization\n", + "- First do a (partial) optimization with first scheme\n", + "- Extract the optimized schemed and adjust its settings\n", + "- Optimize again with the second scheme" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.optimization.optimize import optimize\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"models/model.yml\")\n", + "parameters = load_parameters(\"models/parameters.yml\")\n", + "\n", + "scheme_no_penalties = Scheme(\n", + " model=model,\n", + " parameters=parameters,\n", + " data={\"dataset1\": dataset},\n", + " optimization_method=\"TrustRegionReflection\",\n", + " maximum_number_function_evaluations=5,\n", + ")\n", + "\n", + "# optimization with first scheme\n", + "result_no_penalties_first_run = optimize(scheme_no_penalties)\n", + "# optimization with second scheme,\n", + "# where we change the maximum number of function evaluations\n", + "scheme2 = result_no_penalties_first_run.get_scheme()\n", + "scheme2.maximum_number_function_evaluations = 2\n", + "result_no_penalties = optimize(scheme2)" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "#### Saving results and creating plots" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "base_results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(\n", + " result_no_penalties_first_run,\n", + " base_results_folder / \"no_penalties_first_run/result.yml\",\n", + " allow_overwrite=True,\n", + ")\n", + "save_result(\n", + " result_no_penalties, base_results_folder / \"no_penalties/result.yml\", allow_overwrite=True\n", + ");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.plotting.plot_overview import plot_simple_overview\n", + "\n", + "plot_simple_overview(result_no_penalties_first_run.data[\"dataset1\"])\n", + "plot_simple_overview(result_no_penalties.data[\"dataset1\"]);" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Analysis using equal area penalties\n", + "\n", + "This time we use equal parameter in our model, allowing us to estimate the relative amplitude between the two species." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.optimization.optimize import optimize\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "parameters = load_parameters(\"models/parameters_equal_area_penalties.yml\")\n", + "model = load_model(\"models/model_equal_area_penalties.yml\")\n", + "\n", + "scheme_equal_area = Scheme(\n", + " model=model,\n", + " parameters=parameters,\n", + " data={\"dataset1\": dataset},\n", + " optimization_method=\"TrustRegionReflection\",\n", + " maximum_number_function_evaluations=7,\n", + ")\n", + "\n", + "result_equal_area_first_run = optimize(scheme_equal_area)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "save_result(\n", + " result_equal_area_first_run,\n", + " base_results_folder / \"with_penalties_first_run/result.yml\",\n", + " allow_overwrite=True,\n", + ");" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "#### Run again with results from first run and save" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "result_equal_area = optimize(result_equal_area_first_run.get_scheme())\n", + "save_result(\n", + " result_equal_area,\n", + " base_results_folder / \"with_penalties/result.yml\",\n", + " allow_overwrite=True,\n", + ");" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "#### Plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.plotting.plot_overview import plot_simple_overview\n", + "\n", + "plot_simple_overview(result_equal_area_first_run.data[\"dataset1\"])\n", + "plot_simple_overview(result_equal_area.data[\"dataset1\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/ex_spectral_constraints/ex_spectral_constraints.py b/pyglotaran_examples/ex_spectral_constraints/ex_spectral_constraints.py deleted file mode 100644 index f1885fc..0000000 --- a/pyglotaran_examples/ex_spectral_constraints/ex_spectral_constraints.py +++ /dev/null @@ -1,46 +0,0 @@ -# %% -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_simple_overview - -DATA_PATH = "data/data.ascii" -MODEL_PATHS = { - "no_penalties": {"model": "models/model.yml", "parameters": "models/parameters.yml"}, - "with_penalties": { - "model": "models/model_equal_area_penalties.yml", - "parameters": "models/parameters_equal_area_penalties.yml", - }, -} - -# %% Setup necessary (output) paths -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") -print(f"- Using folder {results_folder} to read/write files for this run") - -# %% Load in data, model and parameters -dataset = load_dataset(script_folder.joinpath(DATA_PATH)) - -for key, val in MODEL_PATHS.items(): - model = load_model(script_folder.joinpath(val["model"])) - parameters = load_parameters(script_folder.joinpath(val["parameters"])) - print(model.markdown(parameters=parameters)) - scheme = Scheme( - model, - parameters, - {"dataset1": dataset}, - maximum_number_function_evaluations=7, # TRF needs at least 6 - ) - result = optimize(scheme) - save_result(result, results_folder / f"{key}_first_run" / "result.yml", allow_overwrite=True) - # Second optimization with results of the first: - scheme2 = result.get_scheme() - result2 = optimize(scheme2) - save_result(result2, results_folder / key / "result.yml", allow_overwrite=True) - plot_simple_overview(result.data["dataset1"], key, figure_only=False) - plot_simple_overview(result2.data["dataset1"], key, figure_only=False) -plt.show() diff --git a/pyglotaran_examples/ex_spectral_constraints/models/model_equal_area_penalties.yml b/pyglotaran_examples/ex_spectral_constraints/models/model_equal_area_penalties.yml index 0acd7e0..7e589f4 100644 --- a/pyglotaran_examples/ex_spectral_constraints/models/model_equal_area_penalties.yml +++ b/pyglotaran_examples/ex_spectral_constraints/models/model_equal_area_penalties.yml @@ -27,7 +27,7 @@ irf: center: irf.center width: irf.width -clp_area_penalties: +clp_penalties: - type: equal_area source: s2 source_intervals: [[0, 1000]] diff --git a/pyglotaran_examples/ex_spectral_guidance/__init__.py b/pyglotaran_examples/ex_spectral_guidance/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyglotaran_examples/ex_spectral_guidance/ex_spectral_guidance.ipynb b/pyglotaran_examples/ex_spectral_guidance/ex_spectral_guidance.ipynb new file mode 100644 index 0000000..af7a663 --- /dev/null +++ b/pyglotaran_examples/ex_spectral_guidance/ex_spectral_guidance.ipynb @@ -0,0 +1,153 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"data/Npq2_220219_800target3fasea.ascii\"),\n", + " \"dataset2\": load_dataset(\"data/trNpq2_220219_800target3fase10SAS5.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"models/model_guidance.yml\")\n", + "parameters = load_parameters(\"models/parameters_guidance.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " # optimization_method=\"Levenberg-Marquardt\", # LM needs 60 nfev!\n", + " maximum_number_function_evaluations=23, # TRF needs nfev=21-23\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "result_plot, _ = plot_overview(result.data[\"dataset1\"], linlog=True, show_data=True)" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "#### Plot Guidance Spectra Fit\n", + "Fit quality and residual" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "guide, _ = plot_overview(result.data[\"dataset2\"], linlog=True, show_data=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/ex_spectral_guidance/ex_spectral_guidance.py b/pyglotaran_examples/ex_spectral_guidance/ex_spectral_guidance.py deleted file mode 100644 index 004c6cf..0000000 --- a/pyglotaran_examples/ex_spectral_guidance/ex_spectral_guidance.py +++ /dev/null @@ -1,82 +0,0 @@ -# %% -from datetime import datetime - -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -DATA_PATH1 = "data/Npq2_220219_800target3fasea.ascii" -DATA_PATH2 = "data/trNpq2_220219_800target3fase10SAS5.ascii" -MODEL_PATH = "models/model_guidance.yml" -PARAMETERS_FILE_PATH = "models/parameters_guidance.yml" - -# %% Setup necessary (output) paths -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") - - -def main(): - # Load in data, model and parameters - parameters = load_parameters(script_folder.joinpath(PARAMETERS_FILE_PATH)) - dataset1 = load_dataset(script_folder.joinpath(DATA_PATH1)) - dataset2 = load_dataset(script_folder.joinpath(DATA_PATH2)) - model = load_model(script_folder.joinpath(MODEL_PATH)) - - # Validate model and parameters - print(model.validate(parameters=parameters)) - - # %% Construct the analysis scheme - scheme = Scheme( - model, - parameters, - {"dataset1": dataset1, "dataset2": dataset2}, - # optimization_method="Levenberg-Marquardt", # LM needs more nfev! - maximum_number_function_evaluations=23, # TRF needs nfev=21-23 - ) - - # Optimize the analysis scheme (and estimate parameters) - result = optimize(scheme) - - # Basic print of results - print(result.markdown(True)) - - return result - - -def load_and_plot_results(): - # Plot and save as PDF - # This set subsequent plots to the glotaran style - plot_style = PlotStyle() - plt.rc("axes", prop_cycle=plot_style.cycler) - - parameter_file = results_folder.joinpath("optimized_parameters.csv") - parameters = load_parameters(str(parameter_file)) - print(f"Optimized parameters:\n {parameters}") - - result1 = results_folder.joinpath("dataset1.nc") - fig1, _ = plot_overview(result1, linlog=True, show_data=True, figure_only=False) - timestamp = datetime.now().strftime("%y%m%d_%H%M") - fig1.savefig( - results_folder.joinpath(f"plot_overview_1of2_{timestamp}.pdf"), bbox_inches="tight" - ) - - result2 = results_folder.joinpath("dataset2.nc") - fig2, _ = plot_overview(result2, linlog=True, figure_only=False) - timestamp = datetime.now().strftime("%y%m%d_%H%M") - fig2.savefig( - results_folder.joinpath(f"plot_overview_2of2_{timestamp}.pdf"), bbox_inches="tight" - ) - plt.show() - - -if __name__ == "__main__": - print(f"- Using folder {results_folder.name} to read/write files for this run") - result = main() - save_result(result, results_folder / "result.yml", allow_overwrite=True) - load_and_plot_results() diff --git a/pyglotaran_examples/ex_two_datasets/__init__.py b/pyglotaran_examples/ex_two_datasets/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyglotaran_examples/ex_two_datasets/ex_two_datasets.ipynb b/pyglotaran_examples/ex_two_datasets/ex_two_datasets.ipynb new file mode 100644 index 0000000..994fde8 --- /dev/null +++ b/pyglotaran_examples/ex_two_datasets/ex_two_datasets.ipynb @@ -0,0 +1,144 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"data/data1.ascii\"),\n", + " \"dataset2\": load_dataset(\"data/data2.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"models/model.yml\")\n", + "parameters = load_parameters(\"models/parameters.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=18,\n", + " optimization_method=\"TrustRegionReflection\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "plot_overview(result.data[\"dataset1\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset2\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/ex_two_datasets/ex_two_datasets.py b/pyglotaran_examples/ex_two_datasets/ex_two_datasets.py deleted file mode 100644 index 0213af1..0000000 --- a/pyglotaran_examples/ex_two_datasets/ex_two_datasets.py +++ /dev/null @@ -1,84 +0,0 @@ -# %% - -from datetime import datetime - -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -DATA_PATH1 = "data/data1.ascii" -DATA_PATH2 = "data/data2.ascii" -MODEL_PATH = "models/model.yml" -PARAMETERS_FILE_PATH = "models/parameters.yml" - -# %% Setup necessary (output) paths -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") -print(f"- Using folder {results_folder.name} to read/write files for this run") - -# %% Load in data, model and parameters -# dataset1 = ExplicitFile(script_folder.joinpath(DATA_PATH1)).read() -# dataset2 = ExplicitFile(script_folder.joinpath(DATA_PATH2)).read() -dataset1 = load_dataset(script_folder.joinpath(DATA_PATH1)) -dataset2 = load_dataset(script_folder.joinpath(DATA_PATH2)) -model = load_model(script_folder.joinpath(MODEL_PATH)) -parameters = load_parameters(script_folder.joinpath(PARAMETERS_FILE_PATH)) - -# %% Validate model and parameters -print(model.validate(parameters=parameters)) - -# %% Construct the analysis scheme -scheme = Scheme( - model, - parameters, - {"dataset1": dataset1, "dataset2": dataset2}, - maximum_number_function_evaluations=18, - optimization_method="TrustRegionReflection", -) - -# %% Optimize the analysis scheme (and estimate parameters) -result = optimize(scheme) -print(result.markdown(True)) -result2 = optimize(result.get_scheme()) -print(result2.markdown(True)) - -# %% Basic print of results -print(result.markdown(True)) - -# %% Save the results -try: - save_result(result, results_folder / "result.yml", allow_overwrite=True) -except (ValueError, FileExistsError) as error: - print(f"catching error: {error}") - try: - save_result(result, results_folder, format_name="yml", allow_overwrite=True) - except FileExistsError as error: - print(f"catching error: {error}") - timestamp = datetime.now().strftime("%y%m%d_%H%M") - save_result( - result_path=str(results_folder.joinpath(timestamp)), - result=result, - format_name="yml", - allow_overwrite=True, - ) - -# %% Plot and save as PDF -# This set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -fig1, _ = plot_overview(result.data["dataset1"], linlog=True, figure_only=False) -timestamp = datetime.now().strftime("%y%m%d_%H%M") -fig1.savefig(results_folder.joinpath(f"plot_overview_1of2_{timestamp}.pdf"), bbox_inches="tight") - -fig2, _ = plot_overview(result.data["dataset2"], linlog=True, figure_only=False) -timestamp = datetime.now().strftime("%y%m%d_%H%M") -fig2.savefig(results_folder.joinpath(f"plot_overview_2of2_{timestamp}.pdf"), bbox_inches="tight") - -plt.show() diff --git a/pyglotaran_examples/ex_two_datasets/models/model.yml b/pyglotaran_examples/ex_two_datasets/models/model.yml index 8247296..3104eec 100644 --- a/pyglotaran_examples/ex_two_datasets/models/model.yml +++ b/pyglotaran_examples/ex_two_datasets/models/model.yml @@ -47,7 +47,7 @@ irf: width: irf.width # It works without clp_area_penalties but then the inputs cannot be estimated -clp_area_penalties: +clp_penalties: - type: equal_area source: s1 source_intervals: [[300, 3000]] diff --git a/pyglotaran_examples/study_fluorescence/__init__.py b/pyglotaran_examples/study_fluorescence/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyglotaran_examples/study_fluorescence/fluorescence_global_and_target_analysis.ipynb b/pyglotaran_examples/study_fluorescence/fluorescence_global_and_target_analysis.ipynb new file mode 100644 index 0000000..80ca1ec --- /dev/null +++ b/pyglotaran_examples/study_fluorescence/fluorescence_global_and_target_analysis.ipynb @@ -0,0 +1,319 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Global and target analysis of time-resolved fluorescence data\n", + "\n", + "From a single experiment on Photosystem I\n", + "\n", + "### Reference\n", + "\n", + "DOI: " + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "### Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# imports\n", + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import load_dataset, load_model, load_parameters, save_result\n", + "from glotaran.optimization.optimize import optimize\n", + "from glotaran.project.scheme import Scheme" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# extra import for plotting\n", + "import matplotlib.pyplot as plt\n", + "from pyglotaran_extras.plotting.plot_overview import plot_overview\n", + "from pyglotaran_extras.plotting.style import PlotStyle" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Location of data files\n", + "\n", + "Note that for the sake of reproduciblity we seperate the model used for global analysis from the model used for target analysis. \n", + "\n", + "We will use the same data files for both analyses.\n", + "\n", + "Note: when comparing the results with previous results, we only use at the target analysis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# paths\n", + "\n", + "GA_MODEL_PATH = \"models/model.yaml\"\n", + "GA_PARAMETERS_PATH = \"models/parameters.yaml\"\n", + "\n", + "results_folder, script_folder = setup_case_study(\n", + " output_folder_name=\"pyglotaran_examples_results\",\n", + ")\n", + "\n", + "data_path = script_folder.joinpath(\"data/data.ascii\")\n", + "model_path = script_folder.joinpath(GA_MODEL_PATH)\n", + "parameter_path = script_folder.joinpath(GA_PARAMETERS_PATH)" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "### Loading in data files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# load the data\n", + "experiment_data = {\"dataset1\": load_dataset(data_path)}\n", + "# load the (model) scheme (library)\n", + "model = load_model(model_path, format_name=\"yml\")\n", + "# load the parameters\n", + "parameters = load_parameters(parameter_path)\n", + "# attach the data to the scheme\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=6, # 6 for TRF, 46 for LM\n", + " # optimization_method=\"Levenberg-Marquardt\", #lm needs nfev=46\n", + ")\n", + "\n", + "scheme.validate()" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "### Optimizing the global fit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Run the optimizer\n", + "result = optimize(scheme, verbose=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "result # shows result as markdown formatted table" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### Inspection of results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect the results\n", + "res = result.data[\"dataset1\"]\n", + "\n", + "res" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Plotting of results\n", + "\n", + "This requires pyglotaran-extras\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot the results\n", + "\n", + "# %% Set subsequent plots to the glotaran style\n", + "plot_style = PlotStyle()\n", + "plt.rc(\"axes\", prop_cycle=plot_style.cycler)\n", + "\n", + "# %%\n", + "fig, _ = plot_overview(res, linlog=False)\n", + "# note species concentration plot still needs work to match styles between the two locatable axis\n", + "\n", + "# %%\n", + "figure_output_path = results_folder / f\"plot_overview_{results_folder.name}.pdf\"\n", + "fig.savefig(str(figure_output_path), bbox_inches=\"tight\")\n", + "print(results_folder)" + ] + }, + { + "cell_type": "markdown", + "id": "15", + "metadata": {}, + "source": [ + "## Target analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "TA_MODEL_PATH = \"models/model-target.yaml\"\n", + "TA_PARAMETERS_PATH = \"models/parameters-target.yaml\"\n", + "\n", + "results_folder, script_folder = setup_case_study(\n", + " output_folder_name=\"pyglotaran_examples_results\",\n", + ")\n", + "results_folder / \"study_fluorescence\"\n", + "\n", + "data_path = script_folder.joinpath(\"data/data.ascii\")\n", + "model_path = script_folder.joinpath(TA_MODEL_PATH)\n", + "parameter_path = script_folder.joinpath(TA_PARAMETERS_PATH)\n", + "\n", + "experiment_data = {\"dataset1\": load_dataset(data_path)}\n", + "model = load_model(model_path, format_name=\"yml\")\n", + "parameters = load_parameters(parameter_path)\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=6, # 6 for TRF, 46 for LM\n", + " # optimization_method=\"Levenberg-Marquardt\", #lm needs nfev=46\n", + ")\n", + "\n", + "result = optimize(scheme, verbose=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "res = result.data[\"dataset1\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot the results\n", + "\n", + "# %% Set subsequent plots to the glotaran style\n", + "plot_style = PlotStyle()\n", + "plt.rc(\"axes\", prop_cycle=plot_style.cycler)\n", + "\n", + "# %%\n", + "fig, _ = plot_overview(res, linlog=False)\n", + "# note species concentration plot still needs work to match styles between the two locatable axis\n", + "\n", + "# %%\n", + "figure_output_path = results_folder / f\"plot_overview_{results_folder.name}.pdf\"\n", + "fig.savefig(str(figure_output_path), bbox_inches=\"tight\")\n", + "print(results_folder)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta_staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/study_fluorescence/target_analysis_script.py b/pyglotaran_examples/study_fluorescence/target_analysis_script.py deleted file mode 100644 index 3d2b574..0000000 --- a/pyglotaran_examples/study_fluorescence/target_analysis_script.py +++ /dev/null @@ -1,86 +0,0 @@ -# To add a new cell, type '# %%' -# To add a new markdown cell, type '# %% [markdown]' -# %% -from timeit import default_timer as timer - -import matplotlib.pyplot as plt -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -GLOBAL_MODEL = "models/model.yaml" -GLOBAL_PARAMS = "models/parameters.yaml" -TARGET_MODEL = "models/model-target.yaml" -TARGET_PARAMS = "models/parameters-target.yaml" -SKIP_FIT = False - -# %% -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") -print(f"Saving results in: {results_folder}") - - -# %% -data_path = script_folder.joinpath("data/data.ascii") -# model_path = script_folder.joinpath(GLOBAL_MODEL) -# parameter_path = script_folder.joinpath(GLOBAL_PARAMS) -model_path = script_folder.joinpath(TARGET_MODEL) -parameter_path = script_folder.joinpath(TARGET_PARAMS) - -print(f"- Using folder {results_folder.name} to read/write files for this run") - -# %% -result_datafile = results_folder.joinpath("dataset1.nc") -if result_datafile.exists() and SKIP_FIT: - print(f"Loading earlier fit results from: {result_datafile}") -else: - dataset = load_dataset(data_path) - model = load_model(model_path) - parameter = load_parameters(parameter_path) - scheme = Scheme( - model, - parameter, - {"dataset1": dataset}, - maximum_number_function_evaluations=6, # 6 for TRF, 46 for LM - # optimization_method="Levenberg-Marquardt", #lm needs nfev=46 - ) - - print(model.validate(parameters=parameter)) - - # %% - start = timer() - # Warning: this may take a while (several seconds per iteration) - result = optimize(scheme, verbose=True) - end = timer() - print(f"Total time: {end - start}") - - save_result(result, results_folder / "result.yml", allow_overwrite=True) - end2 = timer() - print(f"Saving took: {end2 - end}") - - # %% - print(result.markdown(True)) - - # %% - res = result.data["dataset1"] - # Tip: print the xarray object to explore its content - print(res) - - -# %% Set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -# %% -fig, _ = plot_overview(result_datafile, linlog=False, figure_only=False) -# note species concentration plot still needs work to match styles between the two locatable axis - -# %% -fig.savefig( - results_folder.joinpath(f"plot_overview_{results_folder.name}.pdf"), bbox_inches="tight" -) diff --git a/pyglotaran_examples/study_transient_absorption/__init__.py b/pyglotaran_examples/study_transient_absorption/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyglotaran_examples/study_transient_absorption/target_analysis_script.py b/pyglotaran_examples/study_transient_absorption/target_analysis_script.py deleted file mode 100644 index e1f769f..0000000 --- a/pyglotaran_examples/study_transient_absorption/target_analysis_script.py +++ /dev/null @@ -1,52 +0,0 @@ -# %% -from datetime import datetime - -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -DATA_PATH = "data/demo_data_Hippius_etal_JPCC2007_111_13988_Figs5_9.ascii" -MODEL_PATH = "models/model.yml" -PARAMETERS_FILE_PATH = "models/parameters.yml" - -# %% Setup necessary (output) paths -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") -results_folder = results_folder / "target_analysis" -print(f"- Using folder {results_folder} to read/write files for this run") - -# %% Load in data, model and parameters -dataset = load_dataset(script_folder.joinpath(DATA_PATH)) -model = load_model(script_folder.joinpath(MODEL_PATH)) -parameters = load_parameters(script_folder.joinpath(PARAMETERS_FILE_PATH)) - -# %% Validate model and parameters -print(model.validate(parameters=parameters)) - -# %% Construct the analysis scheme -scheme = Scheme(model, parameters, {"dataset1": dataset}, maximum_number_function_evaluations=10) - -# %% Optimize the analysis scheme (and estimate parameters) -result = optimize(scheme) - -# %% Basic print of results -print(result.markdown(True)) - -# %% Save the results -save_result(result, results_folder / "result.yml", allow_overwrite=True) - -# %% Plot and save as PDF -# This set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -fig, _ = plot_overview(result, linlog=True, figure_only=False) - -timestamp = datetime.now().strftime("%y%m%d_%H%M") -fig.savefig(results_folder.joinpath(f"plot_overview_{timestamp}.pdf"), bbox_inches="tight") diff --git a/pyglotaran_examples/study_transient_absorption/transient_absorption_target_analysis.ipynb b/pyglotaran_examples/study_transient_absorption/transient_absorption_target_analysis.ipynb new file mode 100644 index 0000000..b74f12f --- /dev/null +++ b/pyglotaran_examples/study_transient_absorption/transient_absorption_target_analysis.ipynb @@ -0,0 +1,132 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"data/demo_data_Hippius_etal_JPCC2007_111_13988_Figs5_9.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"models/model.yml\")\n", + "parameters = load_parameters(\"models/parameters.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=10,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"target_analysis/result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "plot_overview(result.data[\"dataset1\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/study_transient_absorption/transient_absorption_two_dataset_analysis.ipynb b/pyglotaran_examples/study_transient_absorption/transient_absorption_two_dataset_analysis.ipynb new file mode 100644 index 0000000..3ebbb44 --- /dev/null +++ b/pyglotaran_examples/study_transient_absorption/transient_absorption_two_dataset_analysis.ipynb @@ -0,0 +1,134 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"data/2016co_tol.ascii\"),\n", + " \"dataset2\": load_dataset(\"data/2016c2o_tol.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"models/model_2d_co_co2.yml\")\n", + "parameters = load_parameters(\"models/parameters_2d_co_co2.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=11,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"two_dataset_analysis/result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "plot_overview(result.data[\"dataset1\"])\n", + "plot_overview(result.data[\"dataset2\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/study_transient_absorption/two_dataset_analysis.py b/pyglotaran_examples/study_transient_absorption/two_dataset_analysis.py deleted file mode 100644 index 29dfc54..0000000 --- a/pyglotaran_examples/study_transient_absorption/two_dataset_analysis.py +++ /dev/null @@ -1,101 +0,0 @@ -# To add a new cell, type '# %%' -# To add a new markdown cell, type '# %% [markdown]' -# %% -from timeit import default_timer as timer - -# Needed for plotting only -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -TARGET_MODEL = "models/model_2d_co_co2.yml" -TARGET_PARAMS = "models/parameters_2d_co_co2.yml" - - -# %% -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") -results_folder = results_folder / "two_dataset_analysis" -print(f"Saving results in: {results_folder}") - -# %% -# Dataset are oc and oc2 consisting of a perylene bisimide chromophore -# substituted with either one or two calix[4]arene units in the N-imide position -data_path1 = script_folder.joinpath("data/2016co_tol.ascii") # oc -data_path2 = script_folder.joinpath("data/2016c2o_tol.ascii") # oc2 -model_path = script_folder.joinpath(TARGET_MODEL) # GLOBAL_MODEL or TARGET_MODEL -parameter_path = script_folder.joinpath(TARGET_PARAMS) # GLOBAL_PARAMS or TARGET_PARAMS - -result_name = str(model_path.stem).replace("model", "result") -output_folder = results_folder.joinpath(result_name) -print(f"- Using folder {output_folder.name} to read/write files for this run") - -result_datafile1 = output_folder.joinpath("dataset1.nc") -result_datafile2 = output_folder.joinpath("dataset2.nc") - -# %% - -dataset1 = load_dataset(data_path1) # CO in toluene -dataset2 = load_dataset(data_path2) # C2O in toluene - -print(dataset1) -print(dataset2) - -# %% -model = load_model(model_path) -parameter = load_parameters(parameter_path) -print(model.validate(parameters=parameter)) - -# %% -start = timer() -scheme = Scheme( - model, - parameter, - {"dataset1": dataset1, "dataset2": dataset2}, - maximum_number_function_evaluations=11, -) -result = optimize(scheme) - -end = timer() -print(f"Total time: {end - start}") - -save_result(result, output_folder / "result.yml", allow_overwrite=True) -end2 = timer() -print(f"Saving took: {end2 - end}") - -# %% -print(result.markdown(True)) - -# %% -res1 = result.data["dataset1"] -res2 = result.data["dataset2"] -print(f"Total time: {end - start}") -pad = "#" * 10 -print(f"{pad} res1 {pad}\n\n") -print(res1) -print(f"{pad} res2 {pad}\n\n") -print(res2) - -# %% Set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -# %% -# TODO: enhance plot_overview to handle multiple datasets -fig1, _ = plot_overview(result_datafile1, linlog=True, linthresh=1, figure_only=False) -fig1.savefig( - output_folder.joinpath(f"plot_overview_{result_name}_d1_{data_path1.stem}.pdf"), - bbox_inches="tight", -) - -fig2, _ = plot_overview(result_datafile2, linlog=True, linthresh=1, figure_only=False) -fig2.savefig( - output_folder.joinpath(f"plot_overview_{result_name}_d2_{data_path2.stem}.pdf"), - bbox_inches="tight", -) diff --git a/pyglotaran_examples/test/__init__.py b/pyglotaran_examples/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyglotaran_examples/test/simultaneous_analysis_3d_disp/model.yml b/pyglotaran_examples/test/simultaneous_analysis_3d_disp/model.yml index 8f18930..c88caad 100644 --- a/pyglotaran_examples/test/simultaneous_analysis_3d_disp/model.yml +++ b/pyglotaran_examples/test/simultaneous_analysis_3d_disp/model.yml @@ -65,7 +65,7 @@ irf: center_dispersion_coefficients: [irf.disp1, irf.disp2] # It works without clp_area_penalties but then the inputs cannot be estimated -clp_area_penalties: +clp_penalties: - type: equal_area source: s1 source_intervals: [[300, 3000]] diff --git a/pyglotaran_examples/test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.ipynb b/pyglotaran_examples/test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.ipynb new file mode 100644 index 0000000..2ebd6ab --- /dev/null +++ b/pyglotaran_examples/test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"equareaIRFdispscalsima.ascii\"),\n", + " \"dataset2\": load_dataset(\"equareaIRFdispscalsimb.ascii\"),\n", + " \"dataset3\": load_dataset(\"equareaIRFdispscalsimc.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"model.yml\")\n", + "parameters = load_parameters(\"parameters.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=5, # TRF needs nfev=4, LM needs nfev=55\n", + " # optimization_method=\"Levenberg-Marquardt\", # LM needs nfev=55\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Optimization\n", + "\n", + "Fitting model to the data according to the scheme, optimizing the parameters.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "plot_overview(result.data[\"dataset1\"], linlog=True);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset2\"], linlog=True);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset3\"], linlog=True);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.py b/pyglotaran_examples/test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.py deleted file mode 100644 index c6601a8..0000000 --- a/pyglotaran_examples/test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.py +++ /dev/null @@ -1,73 +0,0 @@ -# %% -# Needed for plotting only -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") - -# read in data -data_path = script_folder.joinpath("equareaIRFdispscalsima.ascii") -dataset1 = load_dataset(data_path) -# print(dataset1) -data_path2 = script_folder.joinpath("equareaIRFdispscalsimb.ascii") -dataset2 = load_dataset(data_path2) -# print(dataset2) -data_path3 = script_folder.joinpath("equareaIRFdispscalsimc.ascii") -dataset3 = load_dataset(data_path3) -# model inlezen + parameters -model_path = script_folder.joinpath("model.yml") -parameters_path = script_folder.joinpath("parameters.yml") - -model = load_model(model_path) - -parameters = load_parameters(parameters_path) - -print(model.validate(parameters=parameters)) - -# define the analysis scheme to optimize -scheme = Scheme( - model, - parameters, - {"dataset1": dataset1, "dataset2": dataset2, "dataset3": dataset3}, - maximum_number_function_evaluations=5, # TRF needs nfev=4, LM needs nfev=55 - # optimization_method="Levenberg-Marquardt", # LM needs nfev=55 -) -# optimize -result = optimize(scheme) -# %% Save results -save_result(result, results_folder / "result.yml", allow_overwrite=True) - -# %% Plot results -# Set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -# TODO: enhance plot_overview to handle multiple datasets -result_datafile1 = results_folder.joinpath("dataset1.nc") -result_datafile2 = results_folder.joinpath("dataset2.nc") -result_datafile3 = results_folder.joinpath("dataset3.nc") -fig1, _ = plot_overview(result_datafile1, linlog=True, linthresh=1, figure_only=False) -fig1.savefig( - results_folder.joinpath("plot_overview_sim3d_d1.pdf"), - bbox_inches="tight", -) - -fig2, _ = plot_overview(result_datafile2, linlog=True, linthresh=1, figure_only=False) -fig2.savefig( - results_folder.joinpath("plot_overview_sim3d_d2.pdf"), - bbox_inches="tight", -) - -fig3, _ = plot_overview(result_datafile3, linlog=True, linthresh=1, figure_only=False) -fig3.savefig( - results_folder.joinpath("plot_overview_sim3d_d3.pdf"), - bbox_inches="tight", -) diff --git a/pyglotaran_examples/test/simultaneous_analysis_3d_nodisp/sim_analysis_script_3d.py b/pyglotaran_examples/test/simultaneous_analysis_3d_nodisp/sim_analysis_script_3d.py deleted file mode 100644 index ad54325..0000000 --- a/pyglotaran_examples/test/simultaneous_analysis_3d_nodisp/sim_analysis_script_3d.py +++ /dev/null @@ -1,72 +0,0 @@ -# %% -# Needed for plotting only -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") - -# read in data -data_path = script_folder.joinpath("equareaIRFsim3a.ascii") -dataset1 = load_dataset(data_path) -# print(dataset1) -data_path2 = script_folder.joinpath("equareaIRFsim3b.ascii") -dataset2 = load_dataset(data_path2) -# print(dataset2) -data_path3 = script_folder.joinpath("equareaIRFsim3c.ascii") -dataset3 = load_dataset(data_path3) -# model inlezen + parameters -model_path = script_folder.joinpath("model.yml") -parameters_path = script_folder.joinpath("parameters.yml") - -model = load_model(model_path) - -parameters = load_parameters(parameters_path) -print(model.validate(parameters=parameters)) - -# define the analysis scheme to optimize -scheme = Scheme( - model, - parameters, - {"dataset1": dataset1, "dataset2": dataset2, "dataset3": dataset3}, - maximum_number_function_evaluations=None, - # optimization_method="Levenberg-Marquardt", -) -# optimize -result = optimize(scheme) -# %% Save results -save_result(result, results_folder / "result.yml", allow_overwrite=True) - -# %% Plot results -# Set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -# TODO: enhance plot_overview to handle multiple datasets -result_datafile1 = results_folder.joinpath("dataset1.nc") -result_datafile2 = results_folder.joinpath("dataset2.nc") -result_datafile3 = results_folder.joinpath("dataset3.nc") -fig1, _ = plot_overview(result_datafile1, linlog=True, linthresh=1, figure_only=False) -fig1.savefig( - results_folder.joinpath("plot_overview_sim3d_d1.pdf"), - bbox_inches="tight", -) - -fig2, _ = plot_overview(result_datafile2, linlog=True, linthresh=1, figure_only=False) -fig2.savefig( - results_folder.joinpath("plot_overview_sim3d_d2.pdf"), - bbox_inches="tight", -) - -fig3, _ = plot_overview(result_datafile3, linlog=True, linthresh=1, figure_only=False) -fig3.savefig( - results_folder.joinpath("plot_overview_sim3d_d3.pdf"), - bbox_inches="tight", -) diff --git a/pyglotaran_examples/test/simultaneous_analysis_3d_nodisp/simultaneous_analysis_3d_nodisp.ipynb b/pyglotaran_examples/test/simultaneous_analysis_3d_nodisp/simultaneous_analysis_3d_nodisp.ipynb new file mode 100644 index 0000000..a8da083 --- /dev/null +++ b/pyglotaran_examples/test/simultaneous_analysis_3d_nodisp/simultaneous_analysis_3d_nodisp.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"equareaIRFsim3a.ascii\"),\n", + " \"dataset2\": load_dataset(\"equareaIRFsim3b.ascii\"),\n", + " \"dataset3\": load_dataset(\"equareaIRFsim3c.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"model.yml\")\n", + "parameters = load_parameters(\"parameters.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=None,\n", + " # optimization_method=\"Levenberg-Marquardt\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Optimization\n", + "\n", + "Fitting model to the data according to the scheme, optimizing the parameters.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "plot_overview(result.data[\"dataset1\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset2\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset3\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/test/simultaneous_analysis_3d_weight/sim_analysis_script_3d_weight.py b/pyglotaran_examples/test/simultaneous_analysis_3d_weight/sim_analysis_script_3d_weight.py deleted file mode 100644 index 7a01cef..0000000 --- a/pyglotaran_examples/test/simultaneous_analysis_3d_weight/sim_analysis_script_3d_weight.py +++ /dev/null @@ -1,73 +0,0 @@ -# %% -# Needed for plotting only -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") - -# read in data -data_path = script_folder.joinpath("equareaIRFsim5.ascii") -dataset1 = load_dataset(data_path) -# print(dataset1) -data_path2 = script_folder.joinpath("equareaIRFsim6.ascii") -dataset2 = load_dataset(data_path2) -# print(dataset2) -data_path3 = script_folder.joinpath("equareaIRFsim8.ascii") -dataset3 = load_dataset(data_path3) -# model inlezen + parameters -model_path = script_folder.joinpath("model.yml") -parameters_path = script_folder.joinpath("parameters.yml") - -model = load_model(model_path) - -parameters = load_parameters(parameters_path) - -print(model.validate(parameters=parameters)) - -# define the analysis scheme to optimize -scheme = Scheme( - model, - parameters, - {"dataset1": dataset1, "dataset2": dataset2, "dataset3": dataset3}, - maximum_number_function_evaluations=111, # 111, #7 for TRF, 86-103 for LM - optimization_method="Levenberg-Marquardt", # lm needs nfev=103 to converge -) -# optimize -result = optimize(scheme) -# %% Save results -save_result(result, results_folder / "result.yml", allow_overwrite=True) - -# %% Plot results -# Set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -# TODO: enhance plot_overview to handle multiple datasets -result_datafile1 = results_folder.joinpath("dataset1.nc") -result_datafile2 = results_folder.joinpath("dataset2.nc") -result_datafile3 = results_folder.joinpath("dataset3.nc") -fig1, _ = plot_overview(result_datafile1, linlog=True, linthresh=1, figure_only=False) -fig1.savefig( - results_folder.joinpath("plot_overview_sim3d_d1.pdf"), - bbox_inches="tight", -) - -fig2, _ = plot_overview(result_datafile2, linlog=True, linthresh=1, figure_only=False) -fig2.savefig( - results_folder.joinpath("plot_overview_sim3d_d2.pdf"), - bbox_inches="tight", -) - -fig3, _ = plot_overview(result_datafile3, linlog=True, linthresh=1, figure_only=False) -fig3.savefig( - results_folder.joinpath("plot_overview_sim3d_d3.pdf"), - bbox_inches="tight", -) diff --git a/pyglotaran_examples/test/simultaneous_analysis_3d_weight/simultaneous_analysis_3d_weight.ipynb b/pyglotaran_examples/test/simultaneous_analysis_3d_weight/simultaneous_analysis_3d_weight.ipynb new file mode 100644 index 0000000..ea763d4 --- /dev/null +++ b/pyglotaran_examples/test/simultaneous_analysis_3d_weight/simultaneous_analysis_3d_weight.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"equareaIRFsim5.ascii\"),\n", + " \"dataset2\": load_dataset(\"equareaIRFsim6.ascii\"),\n", + " \"dataset3\": load_dataset(\"equareaIRFsim8.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"model.yml\")\n", + "parameters = load_parameters(\"parameters.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=111, # 111, #7 for TRF, 86-103 for LM\n", + " optimization_method=\"Levenberg-Marquardt\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Optimization\n", + "\n", + "Fitting model to the data according to the scheme, optimizing the parameters.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "plot_overview(result.data[\"dataset1\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset2\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset3\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyglotaran_examples/test/simultaneous_analysis_6d_disp/sim_analysis_script_6d_disp.py b/pyglotaran_examples/test/simultaneous_analysis_6d_disp/sim_analysis_script_6d_disp.py deleted file mode 100644 index dae39a1..0000000 --- a/pyglotaran_examples/test/simultaneous_analysis_6d_disp/sim_analysis_script_6d_disp.py +++ /dev/null @@ -1,107 +0,0 @@ -# %% -# Needed for plotting only -import matplotlib.pyplot as plt # 3.3 or higher -from glotaran.io import load_dataset -from glotaran.io import load_model -from glotaran.io import load_parameters -from glotaran.io import save_result -from glotaran.optimization.optimize import optimize -from glotaran.project.scheme import Scheme -from pyglotaran_extras.io import setup_case_study -from pyglotaran_extras.plotting.plot_overview import plot_overview -from pyglotaran_extras.plotting.style import PlotStyle - -results_folder, script_folder = setup_case_study(output_folder_name="pyglotaran_examples_results") - -# data inlezen -data_path = script_folder.joinpath("equareaIRFdispscalsim6a.ascii") -dataset1 = load_dataset(data_path) -data_path2 = script_folder.joinpath("equareaIRFdispscalsim6b.ascii") -dataset2 = load_dataset(data_path2) -# print(dataset2) -data_path3 = script_folder.joinpath("equareaIRFdispscalsim6c.ascii") -dataset3 = load_dataset(data_path3) -data_path = script_folder.joinpath("equareaIRFdispscalsim6d.ascii") -dataset4 = load_dataset(data_path) -data_path = script_folder.joinpath("equareaIRFdispscalsim6e.ascii") -dataset5 = load_dataset(data_path) -data_path = script_folder.joinpath("equareaIRFdispscalsim6f.ascii") -dataset6 = load_dataset(data_path) - -# model inlezen + parameters -model_path = script_folder.joinpath("model.yml") -parameters_path = script_folder.joinpath("parameters.yml") - -model = load_model(model_path) - -parameter = load_parameters(parameters_path) - -print(model.validate(parameters=parameter)) - -# analysis schema definieren -scheme = Scheme( - model, - parameter, - { - "dataset1": dataset1, - "dataset2": dataset2, - "dataset3": dataset3, - "dataset4": dataset4, - "dataset5": dataset5, - "dataset6": dataset6, - }, - maximum_number_function_evaluations=9, # TRF needs 8m LM needs 127 - # optimization_method="Levenberg-Marquardt", #LM needs nfev=127 -) -# optimize -result = optimize(scheme) -# %% -# evt opslaan - - -save_result(result, results_folder / "result.yml", allow_overwrite=True) -# evt plotten -# %% Set subsequent plots to the glotaran style -plot_style = PlotStyle() -plt.rc("axes", prop_cycle=plot_style.cycler) - -# %% -# TODO: enhance plot_overview to handle multiple datasets -result_datafile1 = results_folder.joinpath("dataset1.nc") -result_datafile2 = results_folder.joinpath("dataset2.nc") -result_datafile3 = results_folder.joinpath("dataset3.nc") -result_datafile4 = results_folder.joinpath("dataset4.nc") -result_datafile5 = results_folder.joinpath("dataset5.nc") -result_datafile6 = results_folder.joinpath("dataset6.nc") -fig1, _ = plot_overview(result_datafile1, linlog=True, linthresh=5, figure_only=False) -fig1.savefig( - results_folder.joinpath("plot_overview_dummy1.pdf"), - bbox_inches="tight", -) - -fig2, _ = plot_overview(result_datafile2, linlog=True, linthresh=5, figure_only=False) -fig2.savefig( - results_folder.joinpath("plot_overview_dummy2.pdf"), - bbox_inches="tight", -) - -fig3, _ = plot_overview(result_datafile3, linlog=True, linthresh=5, figure_only=False) -fig3.savefig( - results_folder.joinpath("plot_overview_dummy3.pdf"), - bbox_inches="tight", -) -fig4, _ = plot_overview(result_datafile4, linlog=True, linthresh=5, figure_only=False) -fig4.savefig( - results_folder.joinpath("plot_overview_dummy4.pdf"), - bbox_inches="tight", -) -fig5, _ = plot_overview(result_datafile5, linlog=True, linthresh=5, figure_only=False) -fig5.savefig( - results_folder.joinpath("plot_overview_dummy5.pdf"), - bbox_inches="tight", -) -fig6, _ = plot_overview(result_datafile6, linlog=True, linthresh=5, figure_only=False) -fig6.savefig( - results_folder.joinpath("plot_overview_dummy6.pdf"), - bbox_inches="tight", -) diff --git a/pyglotaran_examples/test/simultaneous_analysis_6d_disp/simultaneous_analysis_6d_disp.ipynb b/pyglotaran_examples/test/simultaneous_analysis_6d_disp/simultaneous_analysis_6d_disp.ipynb new file mode 100644 index 0000000..f2ca538 --- /dev/null +++ b/pyglotaran_examples/test/simultaneous_analysis_6d_disp/simultaneous_analysis_6d_disp.ipynb @@ -0,0 +1,190 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "### Load data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_dataset\n", + "\n", + "experiment_data = {\n", + " \"dataset1\": load_dataset(\"equareaIRFdispscalsim6a.ascii\"),\n", + " \"dataset2\": load_dataset(\"equareaIRFdispscalsim6b.ascii\"),\n", + " \"dataset3\": load_dataset(\"equareaIRFdispscalsim6c.ascii\"),\n", + " \"dataset4\": load_dataset(\"equareaIRFdispscalsim6d.ascii\"),\n", + " \"dataset5\": load_dataset(\"equareaIRFdispscalsim6e.ascii\"),\n", + " \"dataset6\": load_dataset(\"equareaIRFdispscalsim6f.ascii\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "### Load model and parameters, define scheme\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.io import load_model, load_parameters\n", + "from glotaran.project.scheme import Scheme\n", + "\n", + "model = load_model(\"model.yml\")\n", + "parameters = load_parameters(\"parameters.yml\")\n", + "\n", + "scheme = Scheme(\n", + " model,\n", + " parameters,\n", + " experiment_data,\n", + " maximum_number_function_evaluations=9, # TRF needs 8m LM needs 127\n", + " # optimization_method=\"Levenberg-Marquardt\", #LM needs nfev=127\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Optimization\n", + "\n", + "Fitting model to the data according to the scheme, optimizing the parameters.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from glotaran.optimization.optimize import optimize\n", + "\n", + "result = optimize(scheme)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras.io import setup_case_study\n", + "\n", + "from glotaran.io import save_result\n", + "\n", + "results_folder, _ = setup_case_study(output_folder_name=\"pyglotaran_examples_results\")\n", + "save_result(result, results_folder / \"result.yml\", allow_overwrite=True);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from pyglotaran_extras import plot_overview\n", + "\n", + "plot_overview(result.data[\"dataset1\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset2\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset3\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset4\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset5\"]);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "plot_overview(result.data[\"dataset6\"]);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pygta-staging", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 68ea1b6..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,25 +0,0 @@ -[tool.black] -line-length = 99 -target-version = ['py38'] -exclude = ''' -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' - -[tool.isort] -profile = "hug" -src_paths = ["pyglotaran_examples", "scripts"] -include_trailing_comma = true -line_length = 99 -force_single_line = true -remove_redundant_aliases = true diff --git a/requirements.txt b/requirements.txt index 4721654..f196561 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -r .github/requirements_ci.txt +yaargh git+https://github.com/glotaran/pyglotaran-extras.git diff --git a/scripts/run_examples.py b/scripts/run_examples.py deleted file mode 100644 index 9966622..0000000 --- a/scripts/run_examples.py +++ /dev/null @@ -1,222 +0,0 @@ -from __future__ import annotations - -import functools -import json -import os -import sys -import warnings -from pathlib import Path - -import yaargh - -REPO_ROOT = Path(__file__).parent.parent - -RESULTS_DIR = REPO_ROOT / "plot_results" -if not RESULTS_DIR.exists(): - RESULTS_DIR.mkdir() - -sys.path.insert(0, str(REPO_ROOT)) - - -def github_format_warning(message, category, filename, lineno, line=None): - return f"::warning file={filename},line={lineno}::{category.__name__}: {message}\n" - - -def save_all_figures(filename: str): - """Save all figures to one PDF""" - import matplotlib.pyplot as plt - from matplotlib.backends.backend_pdf import PdfPages - - result_file = RESULTS_DIR / filename - pp = PdfPages(result_file) - [plt.figure(n).savefig(pp, format="pdf") for n in plt.get_fignums()] - pp.close() - [plt.close(plt.figure(n)) for n in plt.get_fignums()] - print(f"Saved plotting result to: {result_file}") - - -def compress_nc_file(file_path: str | Path): - """Rewrite *.nc files with activated compression.""" - import xarray as xr - - ds = xr.load_dataset(file_path) - comp = {"zlib": True, "complevel": 5} - encoding = {var: comp for var in ds.data_vars} - ds.to_netcdf(file_path, encoding=encoding) - - -def compress_all_results(): - """Rewrite all *.nc result files with activated compression.""" - results_path = Path.home() / "pyglotaran_examples_results" - - for data_file in results_path.rglob("*.nc"): - compress_nc_file(data_file) - - -def script_run_wrapper(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - from glotaran.deprecation.deprecation_utils import GlotaranApiDeprecationWarning - from pyglotaran_extras.deprecation.deprecation_utils import ( - PyglotaranExtrasApiDeprecationWarning, - ) - - print("\n", "#" * 80, sep="") - print("#", f"RUNNING: {func.__name__.upper()}".center(78), "#", sep="") - print("#" * 80, "\n") - if kwargs["headless"]: - import matplotlib - - matplotlib.use("Agg") - warnings.filterwarnings( - "ignore", message=r"Matplotlib.+non-GUI.+", category=UserWarning - ) - if kwargs["raise_on_deprecation"]: - warnings.filterwarnings("error", category=GlotaranApiDeprecationWarning) - warnings.filterwarnings("error", category=PyglotaranExtrasApiDeprecationWarning) - else: - warnings.filterwarnings("always", category=GlotaranApiDeprecationWarning) - warnings.filterwarnings("always", category=PyglotaranExtrasApiDeprecationWarning) - if "GITHUB" in os.environ: - warnings.formatwarning = github_format_warning - - func(*args, **kwargs) - compress_all_results() - - if kwargs["headless"]: - save_all_figures(f"{func.__name__}.pdf") - - return wrapper - - -@script_run_wrapper -def fluorescence(*, headless=False, raise_on_deprecation=False): - """Runs target_analysis_script.py - from pyglotaran_examples/study_fluorescence/""" - # The whole script is run at import. - from pyglotaran_examples.study_fluorescence import target_analysis_script - - -@script_run_wrapper -def transient_absorption(*, headless=False, raise_on_deprecation=False): - """Runs target_analysis_script.py - from pyglotaran_examples/study_transient_absorption""" - # The whole script is run at import. - from pyglotaran_examples.study_transient_absorption import target_analysis_script - - -@script_run_wrapper -def transient_absorption_two_datasets(*, headless=False, raise_on_deprecation=False): - """Runs two_dataset_analysis.py - from pyglotaran_examples/study_transient_absorption""" - # The whole script is run at import. - from pyglotaran_examples.study_transient_absorption import two_dataset_analysis - - -@script_run_wrapper -def spectral_constraints(*, headless=False, raise_on_deprecation=False): - """Runs ex_spectral_constraints.py - from pyglotaran_examples/ex_spectral_constraints""" - # The whole script is run at import. - from pyglotaran_examples.ex_spectral_constraints import ex_spectral_constraints - - -@script_run_wrapper -def spectral_guidance(*, headless=False, raise_on_deprecation=False): - """Runs ex_spectral_guidance.py - from pyglotaran_examples/ex_spectral_guidance""" - # import functions protected by if-name-main and run them - from glotaran.io import save_result - - from pyglotaran_examples.ex_spectral_guidance import ex_spectral_guidance - - result = ex_spectral_guidance.main() - save_result(result, ex_spectral_guidance.results_folder / "result.yml", allow_overwrite=True) - ex_spectral_guidance.load_and_plot_results() - - -@script_run_wrapper -def two_datasets(*, headless=False, raise_on_deprecation=False): - """Runs ex_two_datasets.py - from pyglotaran_examples/ex_two_datasets""" - # The whole script is run at import. - from pyglotaran_examples.ex_two_datasets import ex_two_datasets - - -@script_run_wrapper -def sim_3d_disp(*, headless=False, raise_on_deprecation=False): - """Runs sim_analysis_script_3d_disp.py - from pyglotaran_examples/test/simultaneous_analysis_3d_disp""" - # The whole script is run at import. - from pyglotaran_examples.test.simultaneous_analysis_3d_disp import sim_analysis_script_3d_disp - - -@script_run_wrapper -def sim_3d_nodisp(*, headless=False, raise_on_deprecation=False): - """Runs sim_analysis_script_3d.py - from pyglotaran_examples/test/simultaneous_analysis_3d_nodisp""" - # The whole script is run at import. - from pyglotaran_examples.test.simultaneous_analysis_3d_nodisp import sim_analysis_script_3d - - -@script_run_wrapper -def sim_3d_weight(*, headless=False, raise_on_deprecation=False): - """Runs sim_analysis_script_3d_weight.py - from pyglotaran_examples/test/simultaneous_analysis_3d_weight""" - # The whole script is run at import. - from pyglotaran_examples.test.simultaneous_analysis_3d_weight import ( - sim_analysis_script_3d_weight, - ) - - -@script_run_wrapper -def sim_6d_disp(*, headless=False, raise_on_deprecation=False): - """Runs sim_analysis_script_6d_disp.py - from pyglotaran_examples/test/simultaneous_analysis_6d_disp""" - # The whole script is run at import. - from pyglotaran_examples.test.simultaneous_analysis_6d_disp import sim_analysis_script_6d_disp - - -@script_run_wrapper -def doas_beta(*, headless=False, raise_on_deprecation=False): - """Runs ex_doas_beta.py - from pyglotaran_examples/ex_doas_beta""" - # The whole script is run at import. - from pyglotaran_examples.ex_doas_beta import ex_doas_beta - - -all_funcs = [ - fluorescence, - transient_absorption, - transient_absorption_two_datasets, - spectral_constraints, - spectral_guidance, - two_datasets, - sim_3d_disp, - sim_3d_nodisp, - sim_3d_weight, - sim_6d_disp, - doas_beta, -] - - -def run_all(*, headless=False, raise_on_deprecation=False): - """Runs all examples.""" - for func in all_funcs: - func(headless=headless, raise_on_deprecation=raise_on_deprecation) - - -def set_gha_example_list_output(): - """Export a list of all examples to an output github in github actions.""" - example_names = [func.__name__.replace("_", "-") for func in all_funcs] - gh_output = Path(os.getenv("GITHUB_OUTPUT", "")) - with gh_output.open("a", encoding="utf8") as f: - f.writelines([f"example-list={json.dumps(example_names)}"]) - - -parser = yaargh.ArghParser() -parser.add_commands([*all_funcs, run_all, set_gha_example_list_output]) - - -if __name__ == "__main__": - parser.dispatch() diff --git a/scripts/run_examples_notebooks.py b/scripts/run_examples_notebooks.py new file mode 100644 index 0000000..190ec7c --- /dev/null +++ b/scripts/run_examples_notebooks.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import json +import os +import warnings +from pathlib import Path + +import papermill as pm +import yaargh + +REPO_ROOT = Path(__file__).parent.parent +EXAMPLES_FOLDER = REPO_ROOT / "pyglotaran_examples" + + +def github_format_warning(message, category, filename, lineno, line=None): + return f"::warning file={filename},line={lineno}::{category.__name__}: {message}\n" + + +def compress_nc_file(file_path: str | Path): + """Rewrite *.nc files with activated compression.""" + import xarray as xr + + ds = xr.load_dataset(file_path) + comp = {"zlib": True, "complevel": 5} + encoding = {var: comp for var in ds.data_vars} + ds.to_netcdf(file_path, encoding=encoding) + + +def compress_all_results(): + """Rewrite all *.nc result files with activated compression.""" + results_path = Path.home() / "pyglotaran_examples_results" + + for data_file in results_path.rglob("*.nc"): + compress_nc_file(data_file) + + +def run_notebook(notebook_path: Path) -> Path: + """Run notebook to update results.""" + print("\n", "#" * 80, sep="") + print("#", f"RUNNING: {notebook_path.name.upper()}".center(78), "#", sep="") + print("#" * 80, "\n") + if "GITHUB_OUTPUT" in os.environ: + warnings.formatwarning = github_format_warning + gh_output = Path(os.getenv("GITHUB_OUTPUT", "")) + with gh_output.open("a", encoding="utf8") as f: + f.writelines([f"notebook-path={notebook_path.as_posix()}"]) + print(f"Setting notebook-path output to {notebook_path.as_posix()}") + pm.execute_notebook(notebook_path, notebook_path, cwd=notebook_path.parent) + compress_all_results() + + +def fluorescence(): + """Run study_fluorescence/global_and_target_analysis.ipynb""" + return run_notebook( + EXAMPLES_FOLDER / "study_fluorescence/fluorescence_global_and_target_analysis.ipynb" + ) + + +def transient_absorption(): + """Runs study_transient_absorption/target_analysis.ipynb""" + return run_notebook( + EXAMPLES_FOLDER / "study_transient_absorption/transient_absorption_target_analysis.ipynb" + ) + + +def transient_absorption_two_datasets(): + """Runs study_transient_absorption/two_dataset_analysis.ipynb""" + return run_notebook( + EXAMPLES_FOLDER + / "study_transient_absorption/transient_absorption_two_dataset_analysis.ipynb" + ) + + +def spectral_constraints(): + """Runs ex_spectral_constraints/ex_spectral_constraints.ipynb""" + return run_notebook(EXAMPLES_FOLDER / "ex_spectral_constraints/ex_spectral_constraints.ipynb") + + +def spectral_guidance(): + """Runs ex_spectral_guidance/ex_spectral_guidance.ipynb""" + return run_notebook(EXAMPLES_FOLDER / "ex_spectral_guidance/ex_spectral_guidance.ipynb") + + +def two_datasets(): + """Runs ex_two_datasets/ex_two_datasets.ipynb""" + return run_notebook(EXAMPLES_FOLDER / "ex_two_datasets/ex_two_datasets.ipynb") + + +def sim_3d_disp(): + """Runs test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.ipynb""" + return run_notebook( + EXAMPLES_FOLDER / "test/simultaneous_analysis_3d_disp/sim_analysis_script_3d_disp.ipynb" + ) + + +def sim_3d_nodisp(): + """Runs test/simultaneous_analysis_3d_nodisp/simultaneous_analysis_3d_nodisp.ipynb""" + return run_notebook( + EXAMPLES_FOLDER + / "test/simultaneous_analysis_3d_nodisp/simultaneous_analysis_3d_nodisp.ipynb" + ) + + +def sim_3d_weight(): + """Runs test/simultaneous_analysis_3d_weight/simultaneous_analysis_3d_weight.ipynb""" + return run_notebook( + EXAMPLES_FOLDER + / "test/simultaneous_analysis_3d_weight/simultaneous_analysis_3d_weight.ipynb" + ) + + +def sim_6d_disp(): + """Runs test/simultaneous_analysis_6d_disp/simultaneous_analysis_6d_disp.ipynb""" + return run_notebook( + EXAMPLES_FOLDER / "test/simultaneous_analysis_6d_disp/simultaneous_analysis_6d_disp.ipynb" + ) + + +def doas_beta(): + """Runs ex_doas_beta/ex_doas_beta.ipynb""" + return run_notebook(EXAMPLES_FOLDER / "ex_doas_beta/ex_doas_beta.ipynb") + + +all_funcs = [ + fluorescence, + transient_absorption, + transient_absorption_two_datasets, + spectral_constraints, + spectral_guidance, + two_datasets, + sim_3d_disp, + sim_3d_nodisp, + sim_3d_weight, + sim_6d_disp, + doas_beta, +] + + +def run_all(): + """Runs all examples.""" + errors = {} + for func in all_funcs: + try: + func() + except Exception as err: + errors[func.__name__] = err + if errors: + for func_name, err in errors.items(): + print("\n", "#" * 80, sep="") + print("#", f"Error running: {func_name.upper()}".center(78), "#", sep="") + print("#" * 80, "\n") + print(err) + print("Failed to run the following examples:") + for func_name in errors: + print(func_name) + + +def set_gha_example_list_output(): + """Export a list of all examples to an output github in github actions.""" + example_names = [func.__name__.replace("_", "-") for func in all_funcs] + gh_output = Path(os.getenv("GITHUB_OUTPUT", "")) + with gh_output.open("a", encoding="utf8") as f: + f.writelines([f"example-list={json.dumps(example_names)}"]) + + +parser = yaargh.ArghParser() +parser.add_commands([*all_funcs, run_all, set_gha_example_list_output]) + + +if __name__ == "__main__": + parser.dispatch() diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index dedc2cd..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -extend-ignore = E231, E203 -max-line-length = 99 -per-file-ignores = - scripts/run_examples.py: F401