From 2a26faf66a665345a6c66de3be5ff2d7a9c887d2 Mon Sep 17 00:00:00 2001 From: CompatHelper Julia Date: Sat, 26 Aug 2023 01:00:00 +0000 Subject: [PATCH 1/5] CompatHelper: bump compat for MLJFlux to 0.3, (keep existing compat) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c5434b2..925bf3b 100644 --- a/Project.toml +++ b/Project.toml @@ -30,7 +30,7 @@ ComputationalResources = "0.3" Flux = "0.13.16, 0.14" MLJBase = "0.20, 0.21" MLJEnsembles = "0.3.3" -MLJFlux = "0.2.10" +MLJFlux = "0.2.10, 0.3" MLJModelInterface = "1" MLUtils = "0.4.2" NaturalSort = "1" From 06abefe0ffb1cf9c96d106f714f9a0f37be6a66d Mon Sep 17 00:00:00 2001 From: Pat Alt <55311242+pat-alt@users.noreply.github.com> Date: Thu, 14 Sep 2023 08:27:35 +0200 Subject: [PATCH 2/5] uh --- test/regression.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/regression.jl b/test/regression.jl index 15f8f2c..fac2998 100644 --- a/test/regression.jl +++ b/test/regression.jl @@ -45,7 +45,7 @@ conformal_models = merge(values(available_models[:regression])...) # Fit/Predict: mach = machine(conf_model, X, y) - fit!(mach; rows=train) + MLJ.fit!(mach; rows=train) @test !isnothing(conf_model.scores) predict(mach, selectrows(X, test)) From dd27afb0998e031fa1143269aaec198f4e48d8a0 Mon Sep 17 00:00:00 2001 From: Pat Alt <55311242+pat-alt@users.noreply.github.com> Date: Thu, 14 Sep 2023 08:29:18 +0200 Subject: [PATCH 3/5] come on --- test/classification.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/classification.jl b/test/classification.jl index 32b28a7..8b1d09e 100644 --- a/test/classification.jl +++ b/test/classification.jl @@ -47,7 +47,7 @@ conformal_models = merge(values(available_models[:classification])...) # Fit/Predict: mach = machine(conf_model, X, y) - fit!(mach; rows=train) + MLJ.fit!(mach; rows=train) @test !isnothing(conf_model.scores) predict(mach, selectrows(X, test)) From 68345ffc7c7b311f5377a754793b7e28c8cc06e9 Mon Sep 17 00:00:00 2001 From: Pat Alt <55311242+pat-alt@users.noreply.github.com> Date: Thu, 14 Sep 2023 10:49:59 +0200 Subject: [PATCH 4/5] docs --- README.ipynb | 434 ++++++++++++++++++++++++++++++++++++++++++++ docs/Manifest.toml | 402 ++++++++++++++++++++++------------------ docs/Project.toml | 1 + docs/src/_intro.qmd | 5 +- 4 files changed, 667 insertions(+), 175 deletions(-) create mode 100644 README.ipynb diff --git a/README.ipynb b/README.ipynb new file mode 100644 index 0000000..196ce01 --- /dev/null +++ b/README.ipynb @@ -0,0 +1,434 @@ +{ + "cells": [ + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "---\n", + "format:\n", + " commonmark:\n", + " variant: '-raw_html'\n", + " wrap: none\n", + "execute:\n", + " freeze: auto\n", + " echo: true\n", + " eval: true\n", + " output: false\n", + "crossref:\n", + " fig-prefix: Figure\n", + " tbl-prefix: Table\n", + "bibliography: 'https://raw.githubusercontent.com/pat-alt/bib/main/bib.bib'\n", + "---" + ], + "id": "1a68c0e0" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![](dev/logo/wide_logo.png)\n", + "\n", + "[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliatrustworthyai.github.io/ConformalPrediction.jl/stable/)\n", + "[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliatrustworthyai.github.io/ConformalPrediction.jl/dev/)\n", + "[![Build Status](https://github.com/juliatrustworthyai/ConformalPrediction.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/juliatrustworthyai/ConformalPrediction.jl/actions/workflows/CI.yml?query=branch%3Amain)\n", + "[![Coverage](https://codecov.io/gh/juliatrustworthyai/ConformalPrediction.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/juliatrustworthyai/ConformalPrediction.jl)\n", + "[![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle)\n", + " [![License](https://img.shields.io/github/license/juliatrustworthyai/ConformalPrediction.jl)](LICENSE) \n", + " [![Package Downloads](https://shields.io/endpoint?url=https://pkgs.genieframework.com/api/v1/badge/ConformalPrediction/)](https://pkgs.genieframework.com?packages=ConformalPrediction) \n", + "\n", + "# ConformalPrediction" + ], + "id": "7a45a074" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| echo: false\n", + "using Pkg; Pkg.activate(\"docs\")\n", + "using Plots\n", + "using TaijaPlotting\n", + "theme(:wong)\n", + "using Random\n", + "Random.seed!(123)\n", + "using ConformalPrediction" + ], + "id": "f4268f3d", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`ConformalPrediction.jl` is a package for Predictive Uncertainty Quantification (UQ) through Conformal Prediction (CP) in Julia. It is designed to work with supervised models trained in [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) [@blaom2020mlj]. Conformal Prediction is easy-to-understand, easy-to-use and model-agnostic and it works under minimal distributional assumptions. \n", + "\n", + "## 🏃 Quick Tour\n", + "\n", + "> First time here? Take a quick interactive [tour](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) to see what this package can do right on [JuliaHub](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl){target=\"_blank\"} (To run the notebook, hit login and then edit). \n", + "\n", + "This [`Pluto.jl`](https://github.com/fonsp/Pluto.jl){target=\"_blank\"} 🎈 notebook won the 2nd Price in the [JuliaCon 2023 Notebook Competition](https://info.juliahub.com/pluto-notebook-winner-23).\n", + "\n", + "### Local Tour\n", + "\n", + "To run the tour locally, just clone this repo and start `Pluto.jl` as follows:\n", + "\n", + "```{.julia}\n", + "] add Pluto\n", + "using Pluto\n", + "Pluto.run()\n", + "```\n", + "\n", + "All notebooks are contained in `docs/pluto`.\n", + "\n", + "## 📖 Background\n", + "\n", + "Don't worry, we're not about to deep-dive into methodology. But just to give you a high-level description of Conformal Prediction (CP) upfront:\n", + "\n", + "> Conformal prediction (a.k.a. conformal inference) is a user-friendly paradigm for creating statistically rigorous uncertainty sets/intervals for the predictions of such models. Critically, the sets are valid in a distribution-free sense: they possess explicit, non-asymptotic guarantees even without distributional assumptions or model assumptions.\n", + ">\n", + "> --- @angelopoulos2021gentle\n", + "\n", + "Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates through repeated sampling or the use of dedicated calibration data. \n", + "\n", + "![Conformal Prediction in action: prediction intervals at varying coverage rates. As coverage grows, so does the width of the prediction interval.](https://raw.githubusercontent.com/pat-alt/pat-alt.github.io/main/blog/posts/conformal-regression/www/medium.gif)\n", + "\n", + "The animation above is lifted from a small blog [post](https://www.paltmeyer.com/blog/posts/conformal-regression/) that introduces Conformal Prediction and this package in the context of regression. It shows how the prediction interval and the test points that it covers varies in size as the user-specified coverage rate changes.\n", + "\n", + "## 🚩 Installation \n", + "\n", + "You can install the latest stable release from the general registry:\n", + "\n", + "```{.julia}\n", + "using Pkg\n", + "Pkg.add(\"ConformalPrediction\")\n", + "```\n", + "\n", + "The development version can be installed as follows:\n", + "\n", + "```{.julia}\n", + "using Pkg\n", + "Pkg.add(url=\"https://github.com/juliatrustworthyai/ConformalPrediction.jl\")\n", + "```\n", + "\n", + "## 🔍 Usage Example \n", + "\n", + "To illustrate the intended use of the package, let's have a quick look at a simple regression problem. We first generate some synthetic data and then determine indices for our training and test data using [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/):" + ], + "id": "f57cee42" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "using MLJ\n", + "\n", + "# Inputs:\n", + "N = 600\n", + "xmax = 3.0\n", + "using Distributions\n", + "d = Uniform(-xmax, xmax)\n", + "X = rand(d, N)\n", + "X = reshape(X, :, 1)\n", + "\n", + "# Outputs:\n", + "noise = 0.5\n", + "fun(X) = sin(X)\n", + "ε = randn(N) .* noise\n", + "y = @.(fun(X)) + ε\n", + "y = vec(y)\n", + "\n", + "# Partition:\n", + "train, test = partition(eachindex(y), 0.4, 0.4, shuffle=true)" + ], + "id": "c4c41fca", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We then import Symbolic Regressor ([`SymbolicRegression.jl`](https://github.com/MilesCranmer/SymbolicRegression.jl)) following the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) procedure." + ], + "id": "3368c957" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "regressor = @load SRRegressor pkg=SymbolicRegression\n", + "model = regressor(\n", + " niterations=50,\n", + " binary_operators=[+, -, *],\n", + " unary_operators=[sin],\n", + ")" + ], + "id": "d02dd42e", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To turn our conventional model into a conformal model, we just need to declare it as such by using `conformal_model` wrapper function. The generated conformal model instance can wrapped in data to create a *machine*. Finally, we proceed by fitting the machine on training data using the generic `fit!` method:" + ], + "id": "5e6b6edc" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "using ConformalPrediction\n", + "conf_model = conformal_model(model)\n", + "mach = machine(conf_model, X, y)\n", + "fit!(mach, rows=train)" + ], + "id": "f89ff4c3", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Predictions can then be computed using the generic `predict` method. The code below produces predictions for the first `n` samples. Each tuple contains the lower and upper bound for the prediction interval. " + ], + "id": "b3713843" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| output: true\n", + "show_first = 5\n", + "Xtest = selectrows(X, test)\n", + "ytest = y[test]\n", + "ŷ = predict(mach, Xtest)\n", + "ŷ[1:show_first]" + ], + "id": "269f00d5", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For simple models like this one, we can call a custom `Plots` recipe on our instance, fit result and data to generate the chart below:" + ], + "id": "b177964b" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| output: true\n", + "\n", + "using Plots\n", + "zoom = 0\n", + "plt = plot(mach.model, mach.fitresult, Xtest, ytest, lw=5, zoom=zoom, observed_lab=\"Test points\")\n", + "xrange = range(-xmax+zoom,xmax-zoom,length=N)\n", + "plot!(plt, xrange, @.(fun(xrange)), lw=2, ls=:dash, colour=:darkorange, label=\"Ground truth\")" + ], + "id": "f1820561", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| eval: false\n", + "#| echo: false\n", + "\n", + "theme(:lime)\n", + "\n", + "Xtest = selectrows(X, test)\n", + "ytest = y[test]\n", + "\n", + "max_z = 5\n", + "anim = @animate for z in 0:0.1:max_z\n", + "\n", + " z = -z\n", + "\n", + " # Test points:\n", + " xleft = -xmax + z\n", + " xright = xmax - z\n", + " global Xtest = vcat(xleft, Xtest, xright)\n", + " yleft = fun(xleft) .+ randn(1) .* noise\n", + " yright = fun(xright) .+ randn(1) .* noise\n", + " global ytest = vcat(yleft, ytest, yright)\n", + "\n", + " # Plot:\n", + " plt = plot(mach.model, mach.fitresult, Xtest, ytest, \n", + " lw=5, zoom=zoom, observed_lab=\"Test points\", dpi=300, legend=false, axis=true,\n", + " size=(800,400))\n", + " xrange = range(-xmax-max_z,xmax+max_z,length=N)\n", + " plot!(plt, xrange, @.(fun(xrange)), \n", + " lw=2, ls=:dash, label=\"Ground truth\", xlim=extrema(xrange), ylim=(-2.0,2.0))\n", + "end\n", + "\n", + "gif(anim, fps=10)" + ], + "id": "3ba27895", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can evaluate the conformal model using the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) workflow with a custom performance measure. You can use either `emp_coverage` for the overall empirical coverage (correctness) or `ssc` for the size-stratified coverage rate (adaptiveness)." + ], + "id": "02837116" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| output: true\n", + "\n", + "_eval = evaluate!(mach; measure=[emp_coverage, ssc], verbosity=0)\n", + "display(_eval)\n", + "println(\"Empirical coverage: $(round(_eval.measurement[1], digits=3))\")\n", + "println(\"SSC: $(round(_eval.measurement[2], digits=3))\")" + ], + "id": "6df6a6c4", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 📚 Read on\n", + "\n", + "If after reading the usage example above you are just left with more questions about the topic, that's normal. Below we have have collected a number of further resources to help you get started with this package and the topic itself:\n", + "\n", + "1. Blog post introducing conformal classifiers: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-prediction/)], [[TDS](https://medium.com/towards-data-science/conformal-prediction-in-julia-351b81309e30)], [[Forem](https://forem.julialang.org/patalt/conformal-prediction-in-julia-h9n)].\n", + "2. Blog post applying CP to a deep learning image classifier: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-image-classifier/)], [[TDS](https://medium.com/towards-data-science/how-to-conformalize-a-deep-image-classifier-14ead4e1a5a0)], [[Forem](https://forem.julialang.org/patalt/how-to-conformalize-a-deep-image-classifier-50p2)].\n", + "3. The package [docs](https://juliatrustworthyai.github.io/ConformalPrediction.jl/dev/) and in particular the [FAQ](https://www.paltmeyer.com/ConformalPrediction.jl/dev/faq/).\n", + "\n", + "### External Resources\n", + "\n", + "- *A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification* by @angelopoulos2021gentle ([pdf](https://arxiv.org/pdf/2107.07511.pdf)).\n", + "- *Predictive inference with the jackknife+* by @barber2021predictive ([pdf](https://projecteuclid.org/journals/annals-of-statistics/volume-49/issue-1/Predictive-inference-with-the-jackknife/10.1214/20-AOS1965.full))\n", + "- *Awesome Conformal Prediction* repository by Valery Manokhin ([repo](https://github.com/valeman/awesome-conformal-prediction)).\n", + "- [Documentation](https://mapie.readthedocs.io/en/latest/index.html) for the Python package MAPIE.\n", + "\n", + "## 🔁 Status \n", + "\n", + "This package is in its early stages of development and therefore still subject to changes to the core architecture and API. \n", + "\n", + "### Implemented Methodologies\n", + "\n", + "The following CP approaches have been implemented:\n", + "\n", + "**Regression**:\n", + "\n", + "- Inductive \n", + "- Naive Transductive \n", + "- Jackknife \n", + "- Jackknife+ \n", + "- Jackknife-minmax\n", + "- CV+\n", + "- CV-minmax\n", + "\n", + "**Classification**:\n", + "\n", + "- Inductive\n", + "- Naive Transductive \n", + "- Adaptive Inductive\n", + "\n", + "The package has been tested for the following supervised models offered by [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/).\n", + "\n", + "**Regression**:" + ], + "id": "51c8d581" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| output: true\n", + "keys(tested_atomic_models[:regression])" + ], + "id": "9b93b4cb", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Classification**:" + ], + "id": "043d50cb" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| output: true\n", + "keys(tested_atomic_models[:classification])" + ], + "id": "0f0aab54", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Implemented Evaluation Metrics\n", + "\n", + "To evaluate conformal predictors we are typically interested in correctness and adaptiveness. The former can be evaluated by looking at the empirical coverage rate, while the latter can be assessed through metrics that address the conditional coverage [@angelopoulos2021gentle]. To this end, the following metrics have been implemented:\n", + "\n", + "- `emp_coverage` (empirical coverage)\n", + "- `ssc` (size-stratified coverage)\n", + "\n", + "There is also a simple `Plots.jl` recipe that can be used to inspect the set sizes. In the regression case, the interval width is stratified into discrete bins for this purpose:" + ], + "id": "738ac9b6" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| output: true\n", + "\n", + "bar(mach.model, mach.fitresult, X)" + ], + "id": "7989833b", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 🛠 Contribute \n", + "\n", + "Contributions are welcome! A good place to start is the [list](https://github.com/juliatrustworthyai/ConformalPrediction.jl/issues) of outstanding issues. For more details, see also the [Contributor's Guide](https://www.paltmeyer.com/ConformalPrediction.jl/dev/contribute/). Please follow the [SciML ColPrac guide](https://github.com/SciML/ColPrac).\n", + "\n", + "## 🙏 Thanks\n", + "\n", + "To build this package I have read and re-read both @angelopoulos2021gentle and @barber2021predictive. The Awesome Conformal Prediction [repository](https://github.com/valeman/awesome-conformal-prediction) [@manokhin2022awesome] has also been a fantastic place to get started. Thanks also to [\\@aangelopoulos](https://github.com/aangelopoulos), [\\@valeman](https://github.com/valeman) and others for actively contributing to discussions on here. Quite a few people have also recently started using and contributing to the package for which I am very grateful. Finally, many thanks to Anthony Blaom ([\\@ablaom](https://github.com/ablaom)) for many helpful discussions about how to interface this package to `MLJ.jl`.\n", + "\n", + "## 🎓 References \n" + ], + "id": "9db36793" + } + ], + "metadata": { + "kernelspec": { + "name": "julia-1.9", + "language": "julia", + "display_name": "Julia 1.9.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 2d0743e..b33bf6f 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.0" +julia_version = "1.9.3" manifest_format = "2.0" -project_hash = "006313002b3cecd4f4a1095930b3c91deb25115f" +project_hash = "40ac88472fdbbe01f0c4807f430a730c0b580121" [[deps.ANSIColoredPrinters]] git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" @@ -111,6 +111,12 @@ git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" version = "0.1.0" +[[deps.AtomsBase]] +deps = ["LinearAlgebra", "PeriodicTable", "Printf", "Requires", "StaticArrays", "Unitful", "UnitfulAtomic"] +git-tree-sha1 = "c9804781ca49261c8eb6ce4b62f171cfa3d900f0" +uuid = "a963bdd2-2df7-4f54-a1ee-49d51e6be12a" +version = "0.3.4" + [[deps.AxisAlgorithms]] deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] git-tree-sha1 = "66771c8d21c8ff5e3a93379480a2307ac36863f7" @@ -164,9 +170,9 @@ version = "0.1.1" [[deps.BetaML]] deps = ["AbstractTrees", "CategoricalArrays", "Combinatorics", "DelimitedFiles", "Distributions", "DocStringExtensions", "ForceImport", "JLD2", "LinearAlgebra", "LoopVectorization", "MLJModelInterface", "PDMats", "PrecompileTools", "Printf", "ProgressMeter", "Random", "Reexport", "StableRNGs", "StaticArrays", "Statistics", "StatsBase", "Test", "Zygote"] -git-tree-sha1 = "f20485c4bdbdb99556fe4705222240f28f378fca" +git-tree-sha1 = "eb431cbd022bc55bf9cd1e21abe89b366270b403" uuid = "024491cd-cc6b-443e-8034-08ea7eb7db2b" -version = "0.10.2" +version = "0.10.3" [[deps.BitFlags]] git-tree-sha1 = "43b1a4a8f797c1cddadf60499a8a077d4af2cd2d" @@ -180,9 +186,9 @@ uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b" version = "0.1.5" [[deps.BufferedStreams]] -git-tree-sha1 = "5bcb75a2979e40b29eb250cb26daab67aa8f97f5" +git-tree-sha1 = "4ae47f9a4b1dc19897d3743ff13685925c5202ec" uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" -version = "1.2.0" +version = "1.2.1" [[deps.BytePairEncoding]] deps = ["DoubleArrayTries", "StructWalk", "TextEncodeBase", "Unicode"] @@ -203,9 +209,9 @@ version = "0.4.2" [[deps.CPUSummary]] deps = ["CpuId", "IfElse", "PrecompileTools", "Static"] -git-tree-sha1 = "89e0654ed8c7aebad6d5ad235d6242c2d737a928" +git-tree-sha1 = "601f7e7b3d36f18790e2caf83a882d88e9b71ff1" uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" -version = "0.2.3" +version = "0.2.4" [[deps.CSV]] deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] @@ -215,9 +221,9 @@ version = "0.10.11" [[deps.CUDA]] deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Preferences", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "35160ef0f03b14768abfd68b830f8e3940e8e0dc" +git-tree-sha1 = "968c1365e2992824c3e7a794e30907483f8469a9" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "4.4.0" +version = "4.4.1" [[deps.CUDA_Driver_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] @@ -239,9 +245,9 @@ version = "0.6.0+0" [[deps.CUDNN_jll]] deps = ["Artifacts", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "c30b29597102341a1ea4c2175c4acae9ae522c9d" +git-tree-sha1 = "75923dce4275ead3799b238e10178a68c07dbd3b" uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" -version = "8.9.2+0" +version = "8.9.4+0" [[deps.Cairo_jll]] deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] @@ -276,19 +282,19 @@ weakdeps = ["JSON", "RecipesBase", "SentinelArrays", "StructTypes"] [[deps.CategoricalDistributions]] deps = ["CategoricalArrays", "Distributions", "Missings", "OrderedCollections", "Random", "ScientificTypes"] -git-tree-sha1 = "da68989f027dcefa74d44a452c9e36af9730a70d" +git-tree-sha1 = "ed760a4fde49997ff9360a780abe6e20175162aa" uuid = "af321ab8-2d2e-40a6-b165-3d674595d28e" -version = "0.1.10" +version = "0.1.11" weakdeps = ["UnicodePlots"] [deps.CategoricalDistributions.extensions] UnivariateFiniteDisplayExt = "UnicodePlots" [[deps.ChainRules]] -deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "Statistics", "StructArrays"] -git-tree-sha1 = "f98ae934cd677d51d2941088849f0bf2f59e6f6e" +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "SparseInverseSubset", "Statistics", "StructArrays", "SuiteSparse"] +git-tree-sha1 = "dbeca245b0680f5393b4e6c40dcead7230ab0b3b" uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "1.53.0" +version = "1.54.0" [[deps.ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] @@ -297,10 +303,10 @@ uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" version = "1.16.0" [[deps.Chemfiles]] -deps = ["Chemfiles_jll", "DocStringExtensions"] -git-tree-sha1 = "6951fe6a535a07041122a3a6860a63a7a83e081e" +deps = ["AtomsBase", "Chemfiles_jll", "DocStringExtensions", "PeriodicTable", "Unitful", "UnitfulAtomic"] +git-tree-sha1 = "82fe5e341c793cb51149d993307da9543824b206" uuid = "46823bd8-5fb3-5f92-9aa0-96921f3dd015" -version = "0.10.40" +version = "0.10.41" [[deps.Chemfiles_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -339,14 +345,10 @@ uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" version = "0.11.4" [[deps.ColorVectorSpace]] -deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "Requires", "Statistics", "TensorCore"] -git-tree-sha1 = "a1f44953f2382ebb937d60dafbe2deea4bd23249" +deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "SpecialFunctions", "Statistics", "TensorCore"] +git-tree-sha1 = "600cc5508d66b78aae350f7accdb58763ac18589" uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" -version = "0.10.0" -weakdeps = ["SpecialFunctions"] - - [deps.ColorVectorSpace.extensions] - SpecialFunctionsExt = "SpecialFunctions" +version = "0.9.10" [[deps.Colors]] deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] @@ -378,7 +380,7 @@ weakdeps = ["Dates", "LinearAlgebra"] [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.2+0" +version = "1.0.5+0" [[deps.CompositionsBase]] git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" @@ -403,16 +405,16 @@ uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" version = "2.2.1" [[deps.ConformalPrediction]] -deps = ["CategoricalArrays", "ChainRules", "Flux", "LazyArtifacts", "LinearAlgebra", "MLJBase", "MLJEnsembles", "MLJFlux", "MLJModelInterface", "MLUtils", "NaturalSort", "Plots", "Serialization", "StatsBase"] -git-tree-sha1 = "d4ce78a2a13fa4880daf7051a39f888509788d7d" +deps = ["CategoricalArrays", "ChainRules", "ComputationalResources", "Flux", "LazyArtifacts", "LinearAlgebra", "MLJBase", "MLJEnsembles", "MLJFlux", "MLJModelInterface", "MLUtils", "NaturalSort", "Plots", "ProgressMeter", "Random", "Serialization", "StatsBase", "Tables"] +path = ".." uuid = "98bfc277-1877-43dc-819b-a3e38c30242f" -version = "0.1.8" +version = "0.1.9" [[deps.ConstructionBase]] deps = ["LinearAlgebra"] -git-tree-sha1 = "fe2838a593b5f776e1597e086dcd47560d94e816" +git-tree-sha1 = "c53fc348ca4d40d7b371e71fd52251839080cbc9" uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.5.3" +version = "1.5.4" weakdeps = ["IntervalSets", "StaticArrays"] [deps.ConstructionBase.extensions] @@ -436,6 +438,22 @@ git-tree-sha1 = "f9d7112bfff8a19a3a4ea4e03a8e6a91fe8456bf" uuid = "150eb455-5306-5404-9cee-2592286d6298" version = "0.6.3" +[[deps.CounterfactualExplanations]] +deps = ["CSV", "CUDA", "CategoricalArrays", "ChainRulesCore", "DataFrames", "DecisionTree", "Distributions", "EvoTrees", "Flux", "LaplaceRedux", "LazyArtifacts", "LinearAlgebra", "Logging", "MLDatasets", "MLJBase", "MLJDecisionTreeInterface", "MLJModels", "MLUtils", "MultivariateStats", "NearestNeighborModels", "PackageExtensionCompat", "Parameters", "PrecompileTools", "ProgressMeter", "Random", "Serialization", "Statistics", "StatsBase", "Tables", "UUIDs", "cuDNN"] +git-tree-sha1 = "6cd46eda67b19a577a14eb8a7287197fcb8e9954" +uuid = "2f13d31b-18db-44c1-bc43-ebaf2cff0be0" +version = "0.1.20" + + [deps.CounterfactualExplanations.extensions] + MPIExt = "MPI" + PythonCallExt = "PythonCall" + RCallExt = "RCall" + + [deps.CounterfactualExplanations.weakdeps] + MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" + PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" + RCall = "6f49c342-dc21-5d91-9882-a32aef131414" + [[deps.CpuId]] deps = ["Markdown"] git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406" @@ -572,9 +590,9 @@ version = "0.6.8" [[deps.DynamicExpressions]] deps = ["Compat", "LinearAlgebra", "LoopVectorization", "MacroTools", "PackageExtensionCompat", "PrecompileTools", "Printf", "Random", "Reexport", "TOML"] -git-tree-sha1 = "4f59922b1b80847959c5b1987437d2fcd83c8788" +git-tree-sha1 = "b9d7bcba8809e749a06e15f58f8fada2705bf422" uuid = "a40a106e-89c9-4ca8-8020-a735e8728b6b" -version = "0.12.3" +version = "0.13.1" [deps.DynamicExpressions.extensions] DynamicExpressionsSymbolicUtilsExt = "SymbolicUtils" @@ -609,9 +627,9 @@ version = "0.3.0" [[deps.EvoTrees]] deps = ["BSON", "CUDA", "CategoricalArrays", "Distributions", "MLJModelInterface", "NetworkLayout", "Random", "RecipesBase", "Statistics", "StatsBase", "Tables"] -git-tree-sha1 = "1b418518c0eb1fd1ef0a6d0bfc8051e6abb1232b" +git-tree-sha1 = "a1fa1d1743478394a0a7188d054b67546e4ca143" uuid = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" -version = "0.15.2" +version = "0.16.1" [[deps.ExceptionUnwrapping]] deps = ["Test"] @@ -699,10 +717,15 @@ version = "0.9.20" uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[deps.FillArrays]] -deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] -git-tree-sha1 = "f372472e8672b1d993e93dada09e23139b509f9e" +deps = ["LinearAlgebra", "Random"] +git-tree-sha1 = "a20eaa3ad64254c61eeb5f230d9306e937405434" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.5.0" +version = "1.6.1" +weakdeps = ["SparseArrays", "Statistics"] + + [deps.FillArrays.extensions] + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStatisticsExt = "Statistics" [[deps.FiniteDiff]] deps = ["ArrayInterface", "LinearAlgebra", "Requires", "Setfield", "SparseArrays"] @@ -838,10 +861,10 @@ uuid = "d2c73de3-f751-5644-a686-071e5b155ba9" version = "0.72.8+0" [[deps.GZip]] -deps = ["Libdl"] -git-tree-sha1 = "039be665faf0b8ae36e089cd694233f5dee3f7d6" +deps = ["Libdl", "Zlib_jll"] +git-tree-sha1 = "6388a2d8e409ce23de7d03a7c73d83c5753b3eb2" uuid = "92fee26a-97fe-5a0c-ad85-20a5f3185b63" -version = "0.5.1" +version = "0.6.1" [[deps.GeoInterface]] deps = ["Extents"] @@ -862,10 +885,10 @@ uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" version = "0.21.0+0" [[deps.Glib_jll]] -deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "d3b3624125c1474292d0d8ed0f65554ac37ddb23" +deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "e94c92c7bf4819685eb80186d51c43e71d4afa17" uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" -version = "2.74.0+2" +version = "2.76.5+0" [[deps.Glob]] git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" @@ -896,10 +919,10 @@ uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" version = "1.0.2" [[deps.HDF5]] -deps = ["Compat", "HDF5_jll", "Libdl", "Mmap", "Random", "Requires", "UUIDs"] -git-tree-sha1 = "c73fdc3d9da7700691848b78c61841274076932a" +deps = ["Compat", "HDF5_jll", "Libdl", "Mmap", "Printf", "Random", "Requires", "UUIDs"] +git-tree-sha1 = "114e20044677badbc631ee6fdc80a67920561a29" uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" -version = "0.16.15" +version = "0.16.16" [[deps.HDF5_jll]] deps = ["Artifacts", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"] @@ -915,9 +938,9 @@ version = "1.0.1" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "cb56ccdd481c0dd7f975ad2b3b62d9eda088f7e2" +git-tree-sha1 = "19e974eced1768fb46fd6020171f2cec06b1edb5" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.9.14" +version = "1.9.15" [[deps.HarfBuzz_jll]] deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] @@ -925,12 +948,6 @@ git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3" uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" version = "2.8.1+1" -[[deps.HistogramThresholding]] -deps = ["ImageBase", "LinearAlgebra", "MappedArrays"] -git-tree-sha1 = "7194dfbb2f8d945abdaf68fa9480a965d6661e69" -uuid = "2c695a8d-9458-5d45-9878-1b8a99cf7853" -version = "0.3.1" - [[deps.HostCPUFeatures]] deps = ["BitTwiddlingConvenienceFunctions", "IfElse", "Libdl", "Static"] git-tree-sha1 = "eb8fed28f4994600e29beef49744639d985a04b2" @@ -986,15 +1003,9 @@ version = "0.6.11" [[deps.ImageBase]] deps = ["ImageCore", "Reexport"] -git-tree-sha1 = "eb49b82c172811fd2c86759fa0553a2221feb909" +git-tree-sha1 = "b51bb8cae22c66d0f6357e3bcb6363145ef20835" uuid = "c817782e-172a-44cc-b673-b171935fbb9e" -version = "0.1.7" - -[[deps.ImageBinarization]] -deps = ["HistogramThresholding", "ImageCore", "LinearAlgebra", "Polynomials", "Reexport", "Statistics"] -git-tree-sha1 = "f5356e7203c4a9954962e3757c08033f2efe578a" -uuid = "cbc4b850-ae4b-5111-9e64-df94c024a13d" -version = "0.3.0" +version = "0.1.5" [[deps.ImageContrastAdjustment]] deps = ["ImageBase", "ImageCore", "ImageTransformations", "Parameters"] @@ -1003,16 +1014,10 @@ uuid = "f332f351-ec65-5f6a-b3d1-319c6670881a" version = "0.3.12" [[deps.ImageCore]] -deps = ["AbstractFFTs", "ColorVectorSpace", "Colors", "FixedPointNumbers", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "PrecompileTools", "Reexport"] -git-tree-sha1 = "fc5d1d3443a124fde6e92d0260cd9e064eba69f8" +deps = ["AbstractFFTs", "ColorVectorSpace", "Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "Reexport"] +git-tree-sha1 = "acf614720ef026d38400b3817614c45882d75500" uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" -version = "0.10.1" - -[[deps.ImageCorners]] -deps = ["ImageCore", "ImageFiltering", "PrecompileTools", "StaticArrays", "StatsBase"] -git-tree-sha1 = "24c52de051293745a9bad7d73497708954562b79" -uuid = "89d5987c-236e-4e32-acd0-25bd6bd87b70" -version = "0.1.3" +version = "0.9.4" [[deps.ImageDistances]] deps = ["Distances", "ImageCore", "ImageMorphology", "LinearAlgebra", "Statistics"] @@ -1022,9 +1027,9 @@ version = "0.2.17" [[deps.ImageFiltering]] deps = ["CatIndices", "ComputationalResources", "DataStructures", "FFTViews", "FFTW", "ImageBase", "ImageCore", "LinearAlgebra", "OffsetArrays", "PrecompileTools", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "TiledIteration"] -git-tree-sha1 = "432ae2b430a18c58eb7eca9ef8d0f2db90bc749c" +git-tree-sha1 = "3447781d4c80dbe6d71d239f7cfb1f8049d4c84f" uuid = "6a3955dd-da59-5b1f-98d4-e7296123deb5" -version = "0.7.8" +version = "0.7.6" [[deps.ImageIO]] deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs"] @@ -1051,10 +1056,10 @@ uuid = "bc367c6b-8a6b-528e-b4bd-a4b897500b49" version = "0.9.9" [[deps.ImageMorphology]] -deps = ["DataStructures", "ImageCore", "LinearAlgebra", "LoopVectorization", "OffsetArrays", "Requires", "TiledIteration"] -git-tree-sha1 = "6f0a801136cb9c229aebea0df296cdcd471dbcd1" +deps = ["ImageCore", "LinearAlgebra", "Requires", "TiledIteration"] +git-tree-sha1 = "e7c68ab3df4a75511ba33fc5d8d9098007b579a8" uuid = "787d08f9-d448-5407-9aad-5290dd7ab264" -version = "0.4.5" +version = "0.3.2" [[deps.ImageQualityIndexes]] deps = ["ImageContrastAdjustment", "ImageCore", "ImageDistances", "ImageFiltering", "LazyModules", "OffsetArrays", "PrecompileTools", "Statistics"] @@ -1064,9 +1069,9 @@ version = "0.3.7" [[deps.ImageSegmentation]] deps = ["Clustering", "DataStructures", "Distances", "Graphs", "ImageCore", "ImageFiltering", "ImageMorphology", "LinearAlgebra", "MetaGraphs", "RegionTrees", "SimpleWeightedGraphs", "StaticArrays", "Statistics"] -git-tree-sha1 = "3ff0ca203501c3eedde3c6fa7fd76b703c336b5f" +git-tree-sha1 = "44664eea5408828c03e5addb84fa4f916132fc26" uuid = "80713f31-8817-5129-9cf8-209ff8fb23e1" -version = "1.8.2" +version = "1.8.1" [[deps.ImageShow]] deps = ["Base64", "ColorSchemes", "FileIO", "ImageBase", "ImageCore", "OffsetArrays", "StackViews"] @@ -1075,16 +1080,16 @@ uuid = "4e3cecfd-b093-5904-9786-8bbb286a6a31" version = "0.3.8" [[deps.ImageTransformations]] -deps = ["AxisAlgorithms", "CoordinateTransformations", "ImageBase", "ImageCore", "Interpolations", "OffsetArrays", "Rotations", "StaticArrays"] -git-tree-sha1 = "7ec124670cbce8f9f0267ba703396960337e54b5" +deps = ["AxisAlgorithms", "ColorVectorSpace", "CoordinateTransformations", "ImageBase", "ImageCore", "Interpolations", "OffsetArrays", "Rotations", "StaticArrays"] +git-tree-sha1 = "8717482f4a2108c9358e5c3ca903d3a6113badc9" uuid = "02fcd773-0e25-5acc-982a-7f6622650795" -version = "0.10.0" +version = "0.9.5" [[deps.Images]] -deps = ["Base64", "FileIO", "Graphics", "ImageAxes", "ImageBase", "ImageBinarization", "ImageContrastAdjustment", "ImageCore", "ImageCorners", "ImageDistances", "ImageFiltering", "ImageIO", "ImageMagick", "ImageMetadata", "ImageMorphology", "ImageQualityIndexes", "ImageSegmentation", "ImageShow", "ImageTransformations", "IndirectArrays", "IntegralArrays", "Random", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "TiledIteration"] -git-tree-sha1 = "d438268ed7a665f8322572be0dabda83634d5f45" +deps = ["Base64", "FileIO", "Graphics", "ImageAxes", "ImageBase", "ImageContrastAdjustment", "ImageCore", "ImageDistances", "ImageFiltering", "ImageIO", "ImageMagick", "ImageMetadata", "ImageMorphology", "ImageQualityIndexes", "ImageSegmentation", "ImageShow", "ImageTransformations", "IndirectArrays", "IntegralArrays", "Random", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "TiledIteration"] +git-tree-sha1 = "5fa9f92e1e2918d9d1243b1131abe623cdf98be7" uuid = "916415d5-f1e6-5110-898d-aaa5f9f070e0" -version = "0.26.0" +version = "0.25.3" [[deps.Imath_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -1196,10 +1201,10 @@ uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c" version = "0.1.5" [[deps.JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.4.1" +version = "1.5.0" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] @@ -1263,15 +1268,15 @@ version = "3.0.0+1" [[deps.LLVM]] deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "8695a49bfe05a2dc0feeefd06b4ca6361a018729" +git-tree-sha1 = "a9d2ce1d5007b1e8f6c5b89c5a31ff8bd146db5c" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "6.1.0" +version = "6.2.1" [[deps.LLVMExtra_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "c35203c1e1002747da220ffc3c0762ce7754b08c" +git-tree-sha1 = "7ca6850ae880cc99b59b88517545f91a52020afa" uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" -version = "0.0.23+0" +version = "0.0.25+0" [[deps.LLVMOpenMP_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -1290,6 +1295,12 @@ git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.3.0" +[[deps.LaplaceRedux]] +deps = ["CSV", "Compat", "ComputationalResources", "DataFrames", "Flux", "LinearAlgebra", "MLJ", "MLJBase", "MLJFlux", "MLJModelInterface", "MLUtils", "Parameters", "ProgressMeter", "Random", "Statistics", "Tables", "Tullio", "Zygote"] +git-tree-sha1 = "ca7a96bd2be5066bb2378b42c0191c672811bfaa" +uuid = "c52c1a26-f7c5-402b-80be-ba1e638ad478" +version = "0.1.3" + [[deps.Latexify]] deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "OrderedCollections", "Printf", "Requires"] git-tree-sha1 = "f428ae552340899a935973270b8d98e5a31c49fe" @@ -1372,10 +1383,10 @@ uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" version = "1.42.0+0" [[deps.Libiconv_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "c7cb1f5d892775ba13767a87c7ada0b980ea0a71" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.16.1+2" +version = "1.17.0+0" [[deps.Libmount_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -1431,9 +1442,9 @@ weakdeps = ["ChainRulesCore", "SparseArrays", "Statistics"] [[deps.LogExpFunctions]] deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] -git-tree-sha1 = "c3ce8e7420b3a6e071e0fe4745f5d4300e37b13f" +git-tree-sha1 = "7d6dd4e9212aebaeed356de34ccf262a3cd415aa" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.3.24" +version = "0.3.26" [deps.LogExpFunctions.extensions] LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" @@ -1450,9 +1461,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] -git-tree-sha1 = "cedb76b37bc5a6c702ade66be44f831fa23c681e" +git-tree-sha1 = "0d097476b6c381ab7906460ef1ef1638fbce1d91" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.0.0" +version = "1.0.2" [[deps.LoopVectorization]] deps = ["ArrayInterface", "ArrayInterfaceCore", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] @@ -1467,9 +1478,9 @@ weakdeps = ["ChainRulesCore", "ForwardDiff", "SpecialFunctions"] [[deps.LossFunctions]] deps = ["Markdown", "Requires", "Statistics"] -git-tree-sha1 = "c2b72b61d2e3489b1f9cae3403226b21ec90c943" +git-tree-sha1 = "df9da07efb9b05ca7ef701acec891ee8f73c99e2" uuid = "30fc2ffe-d236-52d8-8643-a9d8f7c094a7" -version = "0.11.0" +version = "0.11.1" weakdeps = ["CategoricalArrays"] [deps.LossFunctions.extensions] @@ -1494,21 +1505,27 @@ version = "2023.2.0+0" [[deps.MLDatasets]] deps = ["CSV", "Chemfiles", "DataDeps", "DataFrames", "DelimitedFiles", "FileIO", "FixedPointNumbers", "GZip", "Glob", "HDF5", "ImageShow", "JLD2", "JSON3", "LazyModules", "MAT", "MLUtils", "NPZ", "Pickle", "Printf", "Requires", "SparseArrays", "Statistics", "Tables"] -git-tree-sha1 = "a03a093b03824f07fe00931df76b18d99398ebb9" +git-tree-sha1 = "10bc70e4c875f1b2ca65cef3ef9ebe705ef936b5" uuid = "eb30cadb-4394-5ae3-aed4-317e484a6458" -version = "0.7.11" +version = "0.7.13" + +[[deps.MLFlowClient]] +deps = ["Dates", "FilePathsBase", "HTTP", "JSON", "ShowCases", "URIs", "UUIDs"] +git-tree-sha1 = "32cee10a6527476bef0c6484ff4c60c2cead5d3e" +uuid = "64a0f543-368b-4a9a-827a-e71edb2a0b83" +version = "0.4.4" [[deps.MLJ]] -deps = ["CategoricalArrays", "ComputationalResources", "Distributed", "Distributions", "LinearAlgebra", "MLJBase", "MLJEnsembles", "MLJIteration", "MLJModels", "MLJTuning", "OpenML", "Pkg", "ProgressMeter", "Random", "ScientificTypes", "Statistics", "StatsBase", "Tables"] -git-tree-sha1 = "d26cd777c711c332019b39445823cbb1f6cdb7e5" +deps = ["CategoricalArrays", "ComputationalResources", "Distributed", "Distributions", "LinearAlgebra", "MLJBase", "MLJEnsembles", "MLJFlow", "MLJIteration", "MLJModels", "MLJTuning", "OpenML", "Pkg", "ProgressMeter", "Random", "Reexport", "ScientificTypes", "Statistics", "StatsBase", "Tables"] +git-tree-sha1 = "193f1f1ac77d91eabe1ac81ff48646b378270eef" uuid = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" -version = "0.19.2" +version = "0.19.5" [[deps.MLJBase]] deps = ["CategoricalArrays", "CategoricalDistributions", "ComputationalResources", "Dates", "DelimitedFiles", "Distributed", "Distributions", "InteractiveUtils", "InvertedIndices", "LinearAlgebra", "LossFunctions", "MLJModelInterface", "Missings", "OrderedCollections", "Parameters", "PrettyTables", "ProgressMeter", "Random", "ScientificTypes", "Serialization", "StatisticalTraits", "Statistics", "StatsBase", "Tables"] -git-tree-sha1 = "2c9d6b9c627a80f6e6acbc6193026f455581fd04" +git-tree-sha1 = "0b7307d1a7214ec3c0ba305571e713f9492ea984" uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d" -version = "0.21.13" +version = "0.21.14" [[deps.MLJDecisionTreeInterface]] deps = ["CategoricalArrays", "DecisionTree", "MLJModelInterface", "Random", "Tables"] @@ -1522,11 +1539,17 @@ git-tree-sha1 = "95b306ef8108067d26dfde9ff3457d59911cc0d6" uuid = "50ed68f4-41fd-4504-931a-ed422449fee0" version = "0.3.3" +[[deps.MLJFlow]] +deps = ["MLFlowClient", "MLJBase", "MLJModelInterface"] +git-tree-sha1 = "bceeeb648c9aa2fc6f65f957c688b164d30f2905" +uuid = "7b7b8358-b45c-48ea-a8ef-7ca328ad328f" +version = "0.1.1" + [[deps.MLJFlux]] deps = ["CategoricalArrays", "ColorTypes", "ComputationalResources", "Flux", "MLJModelInterface", "Metalhead", "ProgressMeter", "Random", "Statistics", "Tables"] -git-tree-sha1 = "b27c3b96cc2a602a1e91eba36b8ca3d796f30ae0" +git-tree-sha1 = "40f9e99a6770bc795f70f1908316e1491488a7b7" uuid = "094fc8d1-fd35-5302-93ea-dabda2abf845" -version = "0.2.10" +version = "0.3.1" [[deps.MLJGLMInterface]] deps = ["Distributions", "GLM", "MLJModelInterface", "StatsModels", "Tables"] @@ -1548,9 +1571,9 @@ version = "0.9.2" [[deps.MLJModelInterface]] deps = ["Random", "ScientificTypesBase", "StatisticalTraits"] -git-tree-sha1 = "c8b7e632d6754a5e36c0d94a4b466a5ba3a30128" +git-tree-sha1 = "03ae109be87f460fe3c96b8a0dbbf9c7bf840bd5" uuid = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" -version = "1.8.0" +version = "1.9.2" [[deps.MLJModels]] deps = ["CategoricalArrays", "CategoricalDistributions", "Combinatorics", "Dates", "Distances", "Distributions", "InteractiveUtils", "LinearAlgebra", "MLJModelInterface", "Markdown", "OrderedCollections", "Parameters", "Pkg", "PrettyPrinting", "REPL", "Random", "RelocatableFolders", "ScientificTypes", "StatisticalTraits", "Statistics", "StatsBase", "Tables"] @@ -1589,9 +1612,9 @@ version = "0.4.3" [[deps.MacroTools]] deps = ["Markdown", "Random"] -git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" +git-tree-sha1 = "9ee1618cbf5240e6d4e0371d6f24065083f60c48" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.10" +version = "0.5.11" [[deps.ManualMemory]] git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd" @@ -1636,10 +1659,10 @@ uuid = "626554b9-1ddb-594c-aa3c-2596fe9399a5" version = "0.7.2" [[deps.Metalhead]] -deps = ["Artifacts", "BSON", "Flux", "Functors", "LazyArtifacts", "MLUtils", "NNlib", "Random", "Statistics"] -git-tree-sha1 = "0e95f91cc5f23610f8f270d7397f307b21e19d2b" +deps = ["Artifacts", "BSON", "CUDA", "ChainRulesCore", "Flux", "Functors", "JLD2", "LazyArtifacts", "MLUtils", "NNlib", "PartialFunctions", "Random", "Statistics"] +git-tree-sha1 = "c093734078e92a4edcf54e850af68ef8cc2c9e03" uuid = "dbeba491-748d-5e0e-a39e-b530a07fa0cc" -version = "0.7.4" +version = "0.8.2" [[deps.MicroCollections]] deps = ["BangBang", "InitialValues", "Setfield"] @@ -1755,9 +1778,9 @@ version = "1.2.0" [[deps.NeuralAttentionlib]] deps = ["Adapt", "CUDA", "ChainRulesCore", "GPUArrays", "GPUArraysCore", "LinearAlgebra", "NNlib", "NNlibCUDA", "Requires", "Static"] -git-tree-sha1 = "dab54e810d7d9159c73d3f8b43de9e4b98286517" +git-tree-sha1 = "a87d94adcbcb0598c147d09b43dce830bc27baaa" uuid = "12afc1b8-fad6-47e1-9132-84abc478905f" -version = "0.2.11" +version = "0.2.12" [[deps.Observables]] git-tree-sha1 = "6862738f9796b3edc1c09d0890afce4eca9e7e93" @@ -1824,9 +1847,9 @@ version = "1.4.1" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "bbb5c2115d63c2f1451cb70e5ef75e8fe4707019" +git-tree-sha1 = "a12e56c72edee3ce6b96667745e6cbbe5498f200" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "1.1.22+0" +version = "1.1.23+0" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] @@ -1842,9 +1865,9 @@ version = "1.7.6" [[deps.Optimisers]] deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "16776280310aa5553c370b9c7b17f34aadaf3c8e" +git-tree-sha1 = "c1fc26bab5df929a5172f296f25d7d08688fd25b" uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" -version = "0.2.19" +version = "0.2.20" [[deps.Opus_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -1903,6 +1926,12 @@ git-tree-sha1 = "b3901ea034cfd8aae57a2fa0dde0b0ea18bad1cb" uuid = "570af359-4316-4cb7-8c74-252c00c2016b" version = "1.1.1" +[[deps.PeriodicTable]] +deps = ["Base64", "Test", "Unitful"] +git-tree-sha1 = "9a9731f346797126271405971dfdf4709947718b" +uuid = "7b2266bf-644c-5ea3-82d8-af4bbd25a884" +version = "1.1.4" + [[deps.Pickle]] deps = ["BFloat16s", "DataStructures", "InternedStrings", "Serialization", "SparseArrays", "Strided", "StringEncodings", "ZipFile"] git-tree-sha1 = "2e71d7dbcab8dc47306c0ed6ac6018fbc1a7070f" @@ -1929,13 +1958,13 @@ version = "0.42.2+0" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.9.0" +version = "1.9.2" [[deps.PkgVersion]] deps = ["Pkg"] -git-tree-sha1 = "f6cf8e7944e50901594838951729a1861e668cb8" +git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" -version = "0.3.2" +version = "0.3.3" [[deps.PlotThemes]] deps = ["PlotUtils", "Statistics"] @@ -1951,9 +1980,9 @@ version = "1.3.5" [[deps.Plots]] deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Preferences", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun", "UnitfulLatexify", "Unzip"] -git-tree-sha1 = "9f8675a55b37a70aa23177ec110f6e3f4dd68466" +git-tree-sha1 = "ccee59c6e48e6f2edf8a5b64dc817b6729f99eb5" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.38.17" +version = "1.39.0" [deps.Plots.extensions] FileIOExt = "FileIO" @@ -1982,18 +2011,20 @@ uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad" version = "0.2.1" [[deps.Polynomials]] -deps = ["LinearAlgebra", "RecipesBase"] -git-tree-sha1 = "3aa2bb4982e575acd7583f01531f241af077b163" +deps = ["LinearAlgebra", "RecipesBase", "Setfield", "SparseArrays"] +git-tree-sha1 = "ea78a2764f31715093de7ab495e12c0187f231d1" uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45" -version = "3.2.13" +version = "4.0.4" [deps.Polynomials.extensions] PolynomialsChainRulesCoreExt = "ChainRulesCore" + PolynomialsFFTWExt = "FFTW" PolynomialsMakieCoreExt = "MakieCore" PolynomialsMutableArithmeticsExt = "MutableArithmetics" [deps.Polynomials.weakdeps] ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" MakieCore = "20f20a25-4f0e-4fdf-b5d1-57303727442b" MutableArithmetics = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" @@ -2011,9 +2042,9 @@ version = "0.2.4" [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.1.2" +version = "1.2.0" [[deps.Preferences]] deps = ["TOML"] @@ -2049,9 +2080,9 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" [[deps.ProgressBars]] deps = ["Printf"] -git-tree-sha1 = "9d84c8646109eb8bc7a006d59b157c64d5155c81" +git-tree-sha1 = "b437cdb0385ed38312d91d9c00c20f3798b30256" uuid = "49802e3a-d2f1-5c88-81d8-b72133a6f568" -version = "1.5.0" +version = "1.5.1" [[deps.ProgressLogging]] deps = ["Logging", "SHA", "UUIDs"] @@ -2061,9 +2092,9 @@ version = "0.1.4" [[deps.ProgressMeter]] deps = ["Distributed", "Printf"] -git-tree-sha1 = "d7a7aef8f8f2d537104f170139553b14dfe39fe9" +git-tree-sha1 = "00099623ffee15972c16111bcf84c58a0051257c" uuid = "92933f4c-e287-5a05-a399-4b506db050ca" -version = "1.7.2" +version = "1.9.0" [[deps.QOI]] deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] @@ -2298,11 +2329,17 @@ version = "1.1.1" deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +[[deps.SparseInverseSubset]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "91402087fd5d13b2d97e3ef29bbdf9d7859e678a" +uuid = "dc90abb0-5640-4711-901d-7e5b23a2fada" +version = "0.1.1" + [[deps.SpecialFunctions]] deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "7beb031cf8145577fbccacd94b8a8f4ce78428d3" +git-tree-sha1 = "e2cfc4012a19088254b3950b85c3c1d8882d864d" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.3.0" +version = "2.3.1" weakdeps = ["ChainRulesCore"] [deps.SpecialFunctions.extensions] @@ -2333,10 +2370,10 @@ uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" version = "0.8.8" [[deps.StaticArrayInterface]] -deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "Requires", "SnoopPrecompile", "SparseArrays", "Static", "SuiteSparse"] -git-tree-sha1 = "33040351d2403b84afce74dae2e22d3f5b18edcb" +deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "PrecompileTools", "Requires", "SparseArrays", "Static", "SuiteSparse"] +git-tree-sha1 = "03fec6800a986d191f64f5c0996b59ed526eda25" uuid = "0d7ed370-da01-4f52-bd93-41d350b8b718" -version = "1.4.0" +version = "1.4.1" weakdeps = ["OffsetArrays", "StaticArrays"] [deps.StaticArrayInterface.extensions] @@ -2345,9 +2382,9 @@ weakdeps = ["OffsetArrays", "StaticArrays"] [[deps.StaticArrays]] deps = ["LinearAlgebra", "Random", "StaticArraysCore"] -git-tree-sha1 = "9cabadf6e7cd2349b6cf49f1915ad2028d65e881" +git-tree-sha1 = "51621cca8651d9e334a659443a74ce50a3b6dfab" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.6.2" +version = "1.6.3" weakdeps = ["Statistics"] [deps.StaticArrays.extensions] @@ -2371,9 +2408,9 @@ version = "1.9.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] -git-tree-sha1 = "45a7769a04a3cf80da1c1c7c60caf932e6f4c9f7" +git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.6.0" +version = "1.7.0" [[deps.StatsBase]] deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] @@ -2396,10 +2433,10 @@ version = "1.3.0" InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" [[deps.StatsModels]] -deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Printf", "REPL", "ShiftedArrays", "SparseArrays", "StatsBase", "StatsFuns", "Tables"] -git-tree-sha1 = "8cc7a5385ecaa420f0b3426f9b0135d0df0638ed" +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Printf", "REPL", "ShiftedArrays", "SparseArrays", "StatsAPI", "StatsBase", "StatsFuns", "Tables"] +git-tree-sha1 = "5cf6c4583533ee38639f73b880f35fc85f2941e0" uuid = "3eaba693-59b7-5ba5-a881-562e759f1c8d" -version = "0.7.2" +version = "0.7.3" [[deps.StatsPlots]] deps = ["AbstractFFTs", "Clustering", "DataStructures", "Distributions", "Interpolations", "KernelDensity", "LinearAlgebra", "MultivariateStats", "NaNMath", "Observables", "Plots", "RecipesBase", "RecipesPipeline", "Reexport", "StatsBase", "TableOperations", "Tables", "Widgets"] @@ -2426,9 +2463,10 @@ uuid = "69024149-9ee7-55f6-a4c4-859efe599b68" version = "0.3.7" [[deps.StringManipulation]] -git-tree-sha1 = "46da2434b41f41ac3594ee9816ce5541c6096123" +deps = ["PrecompileTools"] +git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.3.0" +version = "0.3.4" [[deps.StringViews]] git-tree-sha1 = "f7b06677eae2571c888fd686ba88047d8738b0e3" @@ -2436,10 +2474,10 @@ uuid = "354b36f9-a18e-4713-926e-db85100087ba" version = "1.3.3" [[deps.StructArrays]] -deps = ["Adapt", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] -git-tree-sha1 = "521a0e828e98bb69042fec1809c1b5a680eb7389" +deps = ["Adapt", "ConstructionBase", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] +git-tree-sha1 = "0a3db38e4cce3c54fe7a71f831cd7b6194a54213" uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" -version = "0.6.15" +version = "0.6.16" [[deps.StructTypes]] deps = ["Dates", "UUIDs"] @@ -2464,9 +2502,9 @@ version = "5.10.1+6" [[deps.SymbolicRegression]] deps = ["Compat", "Dates", "Distributed", "DynamicExpressions", "DynamicQuantities", "LineSearches", "LossFunctions", "MLJModelInterface", "MacroTools", "Optim", "PackageExtensionCompat", "Pkg", "PrecompileTools", "Printf", "ProgressBars", "Random", "Reexport", "SpecialFunctions", "StatsBase", "TOML", "Tricks"] -git-tree-sha1 = "83d12323cc7cd5b9800cb0c27e5a7b0fdda58438" +git-tree-sha1 = "185477ccf523aa53c7cf889d6171fbfc63a4bb6c" uuid = "8254be44-1295-4e6a-a16d-46603ac705cb" -version = "0.22.2" +version = "0.22.4" [deps.SymbolicRegression.extensions] SymbolicRegressionJSON3Ext = "JSON3" @@ -2499,6 +2537,12 @@ git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" version = "1.10.1" +[[deps.TaijaPlotting]] +deps = ["CategoricalArrays", "ConformalPrediction", "CounterfactualExplanations", "Flux", "LaplaceRedux", "LinearAlgebra", "MLJBase", "MultivariateStats", "NaturalSort", "Plots"] +git-tree-sha1 = "d5d1c9fccd05c4ff9793394c56fc07c81db40eda" +uuid = "bd7198b4-c7d6-400c-9bab-9a24614b0240" +version = "1.0.2" + [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" @@ -2533,10 +2577,10 @@ uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" version = "0.6.4" [[deps.TiledIteration]] -deps = ["OffsetArrays", "StaticArrayInterface"] -git-tree-sha1 = "1176cc31e867217b06928e2f140c90bd1bc88283" +deps = ["OffsetArrays"] +git-tree-sha1 = "5683455224ba92ef59db72d10690690f4a8dc297" uuid = "06e1c1a7-607b-532d-9fad-de7d9aa2abac" -version = "0.5.0" +version = "0.3.1" [[deps.TimerOutputs]] deps = ["ExprTools", "Printf"] @@ -2581,10 +2625,16 @@ git-tree-sha1 = "aadb748be58b492045b4f56166b5188aa63ce549" uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775" version = "0.1.7" +[[deps.Tullio]] +deps = ["ChainRulesCore", "DiffRules", "LinearAlgebra", "Requires"] +git-tree-sha1 = "7871a39eac745697ee512a87eeff06a048a7905b" +uuid = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" +version = "0.3.5" + [[deps.TupleTools]] -git-tree-sha1 = "3c712976c47707ff893cf6ba4354aa14db1d8938" +git-tree-sha1 = "c8cdc29448afa1a306419f5d1c7af0854c171c80" uuid = "9d95972d-f1c8-5527-a6e0-b4b365fa01f6" -version = "1.3.0" +version = "1.4.1" [[deps.URIs]] git-tree-sha1 = "b7a5e99f24892b6824a954199a45e9ffcc1c70f0" @@ -2632,9 +2682,9 @@ version = "3.6.0" [[deps.Unitful]] deps = ["Dates", "LinearAlgebra", "Random"] -git-tree-sha1 = "64eb17acef1d9734cf09967539818f38093d9b35" +git-tree-sha1 = "a72d22c7e13fe2de562feda8645aa134712a87ee" uuid = "1986cc42-f94f-5a68-af5c-568840ba703d" -version = "1.16.2" +version = "1.17.0" [deps.Unitful.extensions] ConstructionBaseUnitfulExt = "ConstructionBase" @@ -2644,6 +2694,12 @@ version = "1.16.2" ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9" InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" +[[deps.UnitfulAtomic]] +deps = ["Unitful"] +git-tree-sha1 = "903be579194534af1c4b4778d1ace676ca042238" +uuid = "a7773ee8-282e-5fa2-be4e-bd808c38a91a" +version = "1.0.0" + [[deps.UnitfulLatexify]] deps = ["LaTeXStrings", "Latexify", "Unitful"] git-tree-sha1 = "e2d817cc500e960fdbafcf988ac8436ba3208bfd" @@ -2720,10 +2776,10 @@ uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" version = "1.6.1" [[deps.XML2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "93c41695bc1c08c46c5899f4fe06d6ead504bb73" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] +git-tree-sha1 = "04a51d15436a572301b5abbb9d099713327e9fc4" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.10.3+0" +version = "2.10.4+0" [[deps.XSLT_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] @@ -2876,9 +2932,9 @@ version = "1.5.5+0" [[deps.Zygote]] deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArrays", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "e2fe78907130b521619bc88408c859a472c4172b" +git-tree-sha1 = "b97c927497c1de55a78dc9030f6068be5d83ef80" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.63" +version = "0.6.64" [deps.Zygote.extensions] ZygoteColorsExt = "Colors" @@ -2898,9 +2954,9 @@ version = "0.2.3" [[deps.cuDNN]] deps = ["CEnum", "CUDA", "CUDNN_jll"] -git-tree-sha1 = "ee79f97d07bf875231559f9b3f2649f34fac140b" +git-tree-sha1 = "5a1ba43303c62f4a09b0d6751422de03424ab0cd" uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" -version = "1.1.0" +version = "1.1.1" [[deps.fzf_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -2923,7 +2979,7 @@ version = "0.15.1+0" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.7.0+0" +version = "5.8.0+0" [[deps.libfdk_aac_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] diff --git a/docs/Project.toml b/docs/Project.toml index c8b9ecf..bf5af76 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -38,5 +38,6 @@ ShiftedArrays = "1277b4bf-5013-50f5-be3d-901d8477a67a" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd" SymbolicRegression = "8254be44-1295-4e6a-a16d-46603ac705cb" +TaijaPlotting = "bd7198b4-c7d6-400c-9bab-9a24614b0240" Transformers = "21ca0261-441d-5938-ace7-c90938fde4d4" UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" diff --git a/docs/src/_intro.qmd b/docs/src/_intro.qmd index 31bda0a..9865fbb 100644 --- a/docs/src/_intro.qmd +++ b/docs/src/_intro.qmd @@ -4,6 +4,7 @@ #| echo: false using Pkg; Pkg.activate("docs") using Plots +using TaijaPlotting theme(:wong) using Random Random.seed!(123) @@ -14,9 +15,9 @@ using ConformalPrediction ## 🏃 Quick Tour -> First time here? Take a quick interactive [tour](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl) to see what this package can do: [![Binder](https://mybinder.org/badge_logo.svg)](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl){target="_blank"} +> First time here? Take a quick interactive [tour](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) to see what this package can do right on [JuliaHub](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl){target="_blank"} (To run the notebook, hit login and then edit). -The button takes you to a [`Pluto.jl`](https://github.com/fonsp/Pluto.jl){target="_blank"} 🎈 notebook hosted on [binder](https://mybinder.org/){target="_blank"}. In my own experience, this may take some time to load, certainly long enough to get yourself a hot beverage ☕. Alternatively, you can run the notebook locally or skip the tour for now and read on below. +This [`Pluto.jl`](https://github.com/fonsp/Pluto.jl){target="_blank"} 🎈 notebook won the 2nd Price in the [JuliaCon 2023 Notebook Competition](https://info.juliahub.com/pluto-notebook-winner-23). ### Local Tour From 1aca3914d2fb3df8b7119e5af54c5dc1fb9e6ded Mon Sep 17 00:00:00 2001 From: pat-alt Date: Thu, 14 Sep 2023 13:22:40 +0200 Subject: [PATCH 5/5] updated README and bumped version --- Project.toml | 2 +- README.ipynb | 434 -------------- README.md | 32 +- .../figure-commonmark/cell-12-output-1.svg | 68 +-- .../figure-commonmark/cell-7-output-1.svg | 545 +++++++++-------- .../docs/src/index/execute-results/md.json | 2 +- .../figure-commonmark/cell-12-output-1.svg | 49 ++ .../figure-commonmark/cell-7-output-1.svg | 553 +++++++++--------- docs/src/index.md | 31 +- .../figure-commonmark/cell-12-output-1.svg | 49 ++ .../figure-commonmark/cell-7-output-1.svg | 553 +++++++++--------- 11 files changed, 981 insertions(+), 1337 deletions(-) delete mode 100644 README.ipynb create mode 100644 _freeze/docs/src/index/figure-commonmark/cell-12-output-1.svg create mode 100644 docs/src/index_files/figure-commonmark/cell-12-output-1.svg diff --git a/Project.toml b/Project.toml index 925bf3b..63f9afe 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ConformalPrediction" uuid = "98bfc277-1877-43dc-819b-a3e38c30242f" authors = ["Patrick Altmeyer"] -version = "0.1.9" +version = "0.1.10" [deps] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/README.ipynb b/README.ipynb deleted file mode 100644 index 196ce01..0000000 --- a/README.ipynb +++ /dev/null @@ -1,434 +0,0 @@ -{ - "cells": [ - { - "cell_type": "raw", - "metadata": {}, - "source": [ - "---\n", - "format:\n", - " commonmark:\n", - " variant: '-raw_html'\n", - " wrap: none\n", - "execute:\n", - " freeze: auto\n", - " echo: true\n", - " eval: true\n", - " output: false\n", - "crossref:\n", - " fig-prefix: Figure\n", - " tbl-prefix: Table\n", - "bibliography: 'https://raw.githubusercontent.com/pat-alt/bib/main/bib.bib'\n", - "---" - ], - "id": "1a68c0e0" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![](dev/logo/wide_logo.png)\n", - "\n", - "[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliatrustworthyai.github.io/ConformalPrediction.jl/stable/)\n", - "[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliatrustworthyai.github.io/ConformalPrediction.jl/dev/)\n", - "[![Build Status](https://github.com/juliatrustworthyai/ConformalPrediction.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/juliatrustworthyai/ConformalPrediction.jl/actions/workflows/CI.yml?query=branch%3Amain)\n", - "[![Coverage](https://codecov.io/gh/juliatrustworthyai/ConformalPrediction.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/juliatrustworthyai/ConformalPrediction.jl)\n", - "[![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle)\n", - " [![License](https://img.shields.io/github/license/juliatrustworthyai/ConformalPrediction.jl)](LICENSE) \n", - " [![Package Downloads](https://shields.io/endpoint?url=https://pkgs.genieframework.com/api/v1/badge/ConformalPrediction/)](https://pkgs.genieframework.com?packages=ConformalPrediction) \n", - "\n", - "# ConformalPrediction" - ], - "id": "7a45a074" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| echo: false\n", - "using Pkg; Pkg.activate(\"docs\")\n", - "using Plots\n", - "using TaijaPlotting\n", - "theme(:wong)\n", - "using Random\n", - "Random.seed!(123)\n", - "using ConformalPrediction" - ], - "id": "f4268f3d", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`ConformalPrediction.jl` is a package for Predictive Uncertainty Quantification (UQ) through Conformal Prediction (CP) in Julia. It is designed to work with supervised models trained in [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) [@blaom2020mlj]. Conformal Prediction is easy-to-understand, easy-to-use and model-agnostic and it works under minimal distributional assumptions. \n", - "\n", - "## 🏃 Quick Tour\n", - "\n", - "> First time here? Take a quick interactive [tour](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) to see what this package can do right on [JuliaHub](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl){target=\"_blank\"} (To run the notebook, hit login and then edit). \n", - "\n", - "This [`Pluto.jl`](https://github.com/fonsp/Pluto.jl){target=\"_blank\"} 🎈 notebook won the 2nd Price in the [JuliaCon 2023 Notebook Competition](https://info.juliahub.com/pluto-notebook-winner-23).\n", - "\n", - "### Local Tour\n", - "\n", - "To run the tour locally, just clone this repo and start `Pluto.jl` as follows:\n", - "\n", - "```{.julia}\n", - "] add Pluto\n", - "using Pluto\n", - "Pluto.run()\n", - "```\n", - "\n", - "All notebooks are contained in `docs/pluto`.\n", - "\n", - "## 📖 Background\n", - "\n", - "Don't worry, we're not about to deep-dive into methodology. But just to give you a high-level description of Conformal Prediction (CP) upfront:\n", - "\n", - "> Conformal prediction (a.k.a. conformal inference) is a user-friendly paradigm for creating statistically rigorous uncertainty sets/intervals for the predictions of such models. Critically, the sets are valid in a distribution-free sense: they possess explicit, non-asymptotic guarantees even without distributional assumptions or model assumptions.\n", - ">\n", - "> --- @angelopoulos2021gentle\n", - "\n", - "Intuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates through repeated sampling or the use of dedicated calibration data. \n", - "\n", - "![Conformal Prediction in action: prediction intervals at varying coverage rates. As coverage grows, so does the width of the prediction interval.](https://raw.githubusercontent.com/pat-alt/pat-alt.github.io/main/blog/posts/conformal-regression/www/medium.gif)\n", - "\n", - "The animation above is lifted from a small blog [post](https://www.paltmeyer.com/blog/posts/conformal-regression/) that introduces Conformal Prediction and this package in the context of regression. It shows how the prediction interval and the test points that it covers varies in size as the user-specified coverage rate changes.\n", - "\n", - "## 🚩 Installation \n", - "\n", - "You can install the latest stable release from the general registry:\n", - "\n", - "```{.julia}\n", - "using Pkg\n", - "Pkg.add(\"ConformalPrediction\")\n", - "```\n", - "\n", - "The development version can be installed as follows:\n", - "\n", - "```{.julia}\n", - "using Pkg\n", - "Pkg.add(url=\"https://github.com/juliatrustworthyai/ConformalPrediction.jl\")\n", - "```\n", - "\n", - "## 🔍 Usage Example \n", - "\n", - "To illustrate the intended use of the package, let's have a quick look at a simple regression problem. We first generate some synthetic data and then determine indices for our training and test data using [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/):" - ], - "id": "f57cee42" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "using MLJ\n", - "\n", - "# Inputs:\n", - "N = 600\n", - "xmax = 3.0\n", - "using Distributions\n", - "d = Uniform(-xmax, xmax)\n", - "X = rand(d, N)\n", - "X = reshape(X, :, 1)\n", - "\n", - "# Outputs:\n", - "noise = 0.5\n", - "fun(X) = sin(X)\n", - "ε = randn(N) .* noise\n", - "y = @.(fun(X)) + ε\n", - "y = vec(y)\n", - "\n", - "# Partition:\n", - "train, test = partition(eachindex(y), 0.4, 0.4, shuffle=true)" - ], - "id": "c4c41fca", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then import Symbolic Regressor ([`SymbolicRegression.jl`](https://github.com/MilesCranmer/SymbolicRegression.jl)) following the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) procedure." - ], - "id": "3368c957" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "regressor = @load SRRegressor pkg=SymbolicRegression\n", - "model = regressor(\n", - " niterations=50,\n", - " binary_operators=[+, -, *],\n", - " unary_operators=[sin],\n", - ")" - ], - "id": "d02dd42e", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To turn our conventional model into a conformal model, we just need to declare it as such by using `conformal_model` wrapper function. The generated conformal model instance can wrapped in data to create a *machine*. Finally, we proceed by fitting the machine on training data using the generic `fit!` method:" - ], - "id": "5e6b6edc" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "using ConformalPrediction\n", - "conf_model = conformal_model(model)\n", - "mach = machine(conf_model, X, y)\n", - "fit!(mach, rows=train)" - ], - "id": "f89ff4c3", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Predictions can then be computed using the generic `predict` method. The code below produces predictions for the first `n` samples. Each tuple contains the lower and upper bound for the prediction interval. " - ], - "id": "b3713843" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| output: true\n", - "show_first = 5\n", - "Xtest = selectrows(X, test)\n", - "ytest = y[test]\n", - "ŷ = predict(mach, Xtest)\n", - "ŷ[1:show_first]" - ], - "id": "269f00d5", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For simple models like this one, we can call a custom `Plots` recipe on our instance, fit result and data to generate the chart below:" - ], - "id": "b177964b" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| output: true\n", - "\n", - "using Plots\n", - "zoom = 0\n", - "plt = plot(mach.model, mach.fitresult, Xtest, ytest, lw=5, zoom=zoom, observed_lab=\"Test points\")\n", - "xrange = range(-xmax+zoom,xmax-zoom,length=N)\n", - "plot!(plt, xrange, @.(fun(xrange)), lw=2, ls=:dash, colour=:darkorange, label=\"Ground truth\")" - ], - "id": "f1820561", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| eval: false\n", - "#| echo: false\n", - "\n", - "theme(:lime)\n", - "\n", - "Xtest = selectrows(X, test)\n", - "ytest = y[test]\n", - "\n", - "max_z = 5\n", - "anim = @animate for z in 0:0.1:max_z\n", - "\n", - " z = -z\n", - "\n", - " # Test points:\n", - " xleft = -xmax + z\n", - " xright = xmax - z\n", - " global Xtest = vcat(xleft, Xtest, xright)\n", - " yleft = fun(xleft) .+ randn(1) .* noise\n", - " yright = fun(xright) .+ randn(1) .* noise\n", - " global ytest = vcat(yleft, ytest, yright)\n", - "\n", - " # Plot:\n", - " plt = plot(mach.model, mach.fitresult, Xtest, ytest, \n", - " lw=5, zoom=zoom, observed_lab=\"Test points\", dpi=300, legend=false, axis=true,\n", - " size=(800,400))\n", - " xrange = range(-xmax-max_z,xmax+max_z,length=N)\n", - " plot!(plt, xrange, @.(fun(xrange)), \n", - " lw=2, ls=:dash, label=\"Ground truth\", xlim=extrema(xrange), ylim=(-2.0,2.0))\n", - "end\n", - "\n", - "gif(anim, fps=10)" - ], - "id": "3ba27895", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can evaluate the conformal model using the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) workflow with a custom performance measure. You can use either `emp_coverage` for the overall empirical coverage (correctness) or `ssc` for the size-stratified coverage rate (adaptiveness)." - ], - "id": "02837116" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| output: true\n", - "\n", - "_eval = evaluate!(mach; measure=[emp_coverage, ssc], verbosity=0)\n", - "display(_eval)\n", - "println(\"Empirical coverage: $(round(_eval.measurement[1], digits=3))\")\n", - "println(\"SSC: $(round(_eval.measurement[2], digits=3))\")" - ], - "id": "6df6a6c4", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 📚 Read on\n", - "\n", - "If after reading the usage example above you are just left with more questions about the topic, that's normal. Below we have have collected a number of further resources to help you get started with this package and the topic itself:\n", - "\n", - "1. Blog post introducing conformal classifiers: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-prediction/)], [[TDS](https://medium.com/towards-data-science/conformal-prediction-in-julia-351b81309e30)], [[Forem](https://forem.julialang.org/patalt/conformal-prediction-in-julia-h9n)].\n", - "2. Blog post applying CP to a deep learning image classifier: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-image-classifier/)], [[TDS](https://medium.com/towards-data-science/how-to-conformalize-a-deep-image-classifier-14ead4e1a5a0)], [[Forem](https://forem.julialang.org/patalt/how-to-conformalize-a-deep-image-classifier-50p2)].\n", - "3. The package [docs](https://juliatrustworthyai.github.io/ConformalPrediction.jl/dev/) and in particular the [FAQ](https://www.paltmeyer.com/ConformalPrediction.jl/dev/faq/).\n", - "\n", - "### External Resources\n", - "\n", - "- *A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification* by @angelopoulos2021gentle ([pdf](https://arxiv.org/pdf/2107.07511.pdf)).\n", - "- *Predictive inference with the jackknife+* by @barber2021predictive ([pdf](https://projecteuclid.org/journals/annals-of-statistics/volume-49/issue-1/Predictive-inference-with-the-jackknife/10.1214/20-AOS1965.full))\n", - "- *Awesome Conformal Prediction* repository by Valery Manokhin ([repo](https://github.com/valeman/awesome-conformal-prediction)).\n", - "- [Documentation](https://mapie.readthedocs.io/en/latest/index.html) for the Python package MAPIE.\n", - "\n", - "## 🔁 Status \n", - "\n", - "This package is in its early stages of development and therefore still subject to changes to the core architecture and API. \n", - "\n", - "### Implemented Methodologies\n", - "\n", - "The following CP approaches have been implemented:\n", - "\n", - "**Regression**:\n", - "\n", - "- Inductive \n", - "- Naive Transductive \n", - "- Jackknife \n", - "- Jackknife+ \n", - "- Jackknife-minmax\n", - "- CV+\n", - "- CV-minmax\n", - "\n", - "**Classification**:\n", - "\n", - "- Inductive\n", - "- Naive Transductive \n", - "- Adaptive Inductive\n", - "\n", - "The package has been tested for the following supervised models offered by [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/).\n", - "\n", - "**Regression**:" - ], - "id": "51c8d581" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| output: true\n", - "keys(tested_atomic_models[:regression])" - ], - "id": "9b93b4cb", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Classification**:" - ], - "id": "043d50cb" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| output: true\n", - "keys(tested_atomic_models[:classification])" - ], - "id": "0f0aab54", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Implemented Evaluation Metrics\n", - "\n", - "To evaluate conformal predictors we are typically interested in correctness and adaptiveness. The former can be evaluated by looking at the empirical coverage rate, while the latter can be assessed through metrics that address the conditional coverage [@angelopoulos2021gentle]. To this end, the following metrics have been implemented:\n", - "\n", - "- `emp_coverage` (empirical coverage)\n", - "- `ssc` (size-stratified coverage)\n", - "\n", - "There is also a simple `Plots.jl` recipe that can be used to inspect the set sizes. In the regression case, the interval width is stratified into discrete bins for this purpose:" - ], - "id": "738ac9b6" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| output: true\n", - "\n", - "bar(mach.model, mach.fitresult, X)" - ], - "id": "7989833b", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 🛠 Contribute \n", - "\n", - "Contributions are welcome! A good place to start is the [list](https://github.com/juliatrustworthyai/ConformalPrediction.jl/issues) of outstanding issues. For more details, see also the [Contributor's Guide](https://www.paltmeyer.com/ConformalPrediction.jl/dev/contribute/). Please follow the [SciML ColPrac guide](https://github.com/SciML/ColPrac).\n", - "\n", - "## 🙏 Thanks\n", - "\n", - "To build this package I have read and re-read both @angelopoulos2021gentle and @barber2021predictive. The Awesome Conformal Prediction [repository](https://github.com/valeman/awesome-conformal-prediction) [@manokhin2022awesome] has also been a fantastic place to get started. Thanks also to [\\@aangelopoulos](https://github.com/aangelopoulos), [\\@valeman](https://github.com/valeman) and others for actively contributing to discussions on here. Quite a few people have also recently started using and contributing to the package for which I am very grateful. Finally, many thanks to Anthony Blaom ([\\@ablaom](https://github.com/ablaom)) for many helpful discussions about how to interface this package to `MLJ.jl`.\n", - "\n", - "## 🎓 References \n" - ], - "id": "9db36793" - } - ], - "metadata": { - "kernelspec": { - "name": "julia-1.9", - "language": "julia", - "display_name": "Julia 1.9.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/README.md b/README.md index 0f75edc..a8de98c 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ ## 🏃 Quick Tour -> First time here? Take a quick interactive [tour](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl) to see what this package can do: [![Binder](https://mybinder.org/badge_logo.svg)](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl) +> First time here? Take a quick interactive [tour](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) to see what this package can do right on [JuliaHub](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) (To run the notebook, hit login and then edit). -The button takes you to a [`Pluto.jl`](https://github.com/fonsp/Pluto.jl) 🎈 notebook hosted on [binder](https://mybinder.org/). In my own experience, this may take some time to load, certainly long enough to get yourself a hot beverage ☕. Alternatively, you can run the notebook locally or skip the tour for now and read on below. +This [`Pluto.jl`](https://github.com/fonsp/Pluto.jl) 🎈 notebook won the 2nd Price in the [JuliaCon 2023 Notebook Competition](https://info.juliahub.com/pluto-notebook-winner-23). ### Local Tour @@ -111,11 +111,11 @@ ŷ[1:show_first] ``` 5-element Vector{Tuple{Float64, Float64}}: - (0.0458889297242715, 1.9182762960257687) - (-1.9174452847238976, -0.04505791842240037) - (-1.2544275358451678, 0.6179598304563294) - (-0.2818835218505735, 1.5905038444509236) - (0.01299565032151917, 1.8853830166230163) + (-0.04087262272113379, 1.8635644669554758) + (0.04647464096907805, 1.9509117306456876) + (-0.24248802236397216, 1.6619490673126376) + (-0.07841928163933476, 1.8260178080372749) + (-0.02268628324126465, 1.881750806435345) For simple models like this one, we can call a custom `Plots` recipe on our instance, fit result and data to generate the chart below: @@ -139,20 +139,20 @@ println("SSC: $(round(_eval.measurement[2], digits=3))") ``` PerformanceEvaluation object with these fields: - measure, operation, measurement, per_fold, + model, measure, operation, measurement, per_fold, per_observation, fitted_params_per_fold, - report_per_fold, train_test_rows + report_per_fold, train_test_rows, resampling, repeats Extract: ┌──────────────────────────────────────────────┬───────────┬─────────────┬────── │ measure │ operation │ measurement │ 1.9 ⋯ ├──────────────────────────────────────────────┼───────────┼─────────────┼────── - │ ConformalPrediction.emp_coverage │ predict │ 0.948 │ 0.0 ⋯ - │ ConformalPrediction.size_stratified_coverage │ predict │ 0.948 │ 0.0 ⋯ + │ ConformalPrediction.emp_coverage │ predict │ 0.953 │ 0.0 ⋯ + │ ConformalPrediction.size_stratified_coverage │ predict │ 0.953 │ 0.0 ⋯ └──────────────────────────────────────────────┴───────────┴─────────────┴────── 2 columns omitted - Empirical coverage: 0.948 - SSC: 0.948 + Empirical coverage: 0.953 + SSC: 0.953 ## 📚 Read on @@ -240,14 +240,14 @@ Contributions are welcome! A good place to start is the [list](https://github.co ## 🙏 Thanks -To build this package I have read and re-read both Angelopoulos and Bates (2021) and Barber et al. (2021). The Awesome Conformal Prediction [repository](https://github.com/valeman/awesome-conformal-prediction) (Manokhin, n.d.) has also been a fantastic place to get started. Thanks also to [@aangelopoulos](https://github.com/aangelopoulos), [@valeman](https://github.com/valeman) and others for actively contributing to discussions on here. Quite a few people have also recently started using and contributing to the package for which I am very grateful. Finally, many thanks to Anthony Blaom ([@ablaom](https://github.com/ablaom)) for many helpful discussions about how to interface this package to `MLJ.jl`. +To build this package I have read and re-read both Angelopoulos and Bates (2021) and Barber et al. (2021). The Awesome Conformal Prediction [repository](https://github.com/valeman/awesome-conformal-prediction) (Manokhin 2022) has also been a fantastic place to get started. Thanks also to [@aangelopoulos](https://github.com/aangelopoulos), [@valeman](https://github.com/valeman) and others for actively contributing to discussions on here. Quite a few people have also recently started using and contributing to the package for which I am very grateful. Finally, many thanks to Anthony Blaom ([@ablaom](https://github.com/ablaom)) for many helpful discussions about how to interface this package to `MLJ.jl`. ## 🎓 References -Angelopoulos, Anastasios N., and Stephen Bates. 2021. “A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification.” . +Angelopoulos, Anastasios N, and Stephen Bates. 2021. “A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification.” *arXiv Preprint arXiv:2107.07511*. Barber, Rina Foygel, Emmanuel J. Candès, Aaditya Ramdas, and Ryan J. Tibshirani. 2021. “Predictive Inference with the Jackknife+.” *The Annals of Statistics* 49 (1): 486–507. . Blaom, Anthony D., Franz Kiraly, Thibaut Lienart, Yiannis Simillides, Diego Arenas, and Sebastian J. Vollmer. 2020. “MLJ: A Julia Package for Composable Machine Learning.” *Journal of Open Source Software* 5 (55): 2704. . -Manokhin, Valery. n.d. “Awesome Conformal Prediction.” +Manokhin, Valery. 2022. “Awesome Conformal Prediction.” Zenodo. . diff --git a/README_files/figure-commonmark/cell-12-output-1.svg b/README_files/figure-commonmark/cell-12-output-1.svg index 0ea75b9..20cb47e 100644 --- a/README_files/figure-commonmark/cell-12-output-1.svg +++ b/README_files/figure-commonmark/cell-12-output-1.svg @@ -1,49 +1,49 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/README_files/figure-commonmark/cell-7-output-1.svg b/README_files/figure-commonmark/cell-7-output-1.svg index b253af4..b943f97 100644 --- a/README_files/figure-commonmark/cell-7-output-1.svg +++ b/README_files/figure-commonmark/cell-7-output-1.svg @@ -1,288 +1,285 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/_freeze/docs/src/index/execute-results/md.json b/_freeze/docs/src/index/execute-results/md.json index 0096fa8..09b922a 100644 --- a/_freeze/docs/src/index/execute-results/md.json +++ b/_freeze/docs/src/index/execute-results/md.json @@ -1,7 +1,7 @@ { "hash": "00c3ee841711180ac18233205c0fd8eb", "result": { - "markdown": "---\ntitle: ConformalPrediction\n---\n\n\n```@meta\nCurrentModule = ConformalPrediction\n```\n\n![](assets/wide_logo.png)\n\nDocumentation for [ConformalPrediction.jl](https://github.com/juliatrustworthyai/ConformalPrediction.jl).\n\n\n\n\n\n`ConformalPrediction.jl` is a package for Predictive Uncertainty Quantification (UQ) through Conformal Prediction (CP) in Julia. It is designed to work with supervised models trained in [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) [@blaom2020mlj]. Conformal Prediction is easy-to-understand, easy-to-use and model-agnostic and it works under minimal distributional assumptions. \n\n## 🏃 Quick Tour\n\n> First time here? Take a quick interactive [tour](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl) to see what this package can do: [![Binder](https://mybinder.org/badge_logo.svg)](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl){target=\"_blank\"}\n\nThe button takes you to a [`Pluto.jl`](https://github.com/fonsp/Pluto.jl){target=\"_blank\"} 🎈 notebook hosted on [binder](https://mybinder.org/){target=\"_blank\"}. In my own experience, this may take some time to load, certainly long enough to get yourself a hot beverage ☕. Alternatively, you can run the notebook locally or skip the tour for now and read on below. \n\n### Local Tour\n\nTo run the tour locally, just clone this repo and start `Pluto.jl` as follows:\n\n```{.julia}\n] add Pluto\nusing Pluto\nPluto.run()\n```\n\nAll notebooks are contained in `docs/pluto`.\n\n## 📖 Background\n\nDon't worry, we're not about to deep-dive into methodology. But just to give you a high-level description of Conformal Prediction (CP) upfront:\n\n> Conformal prediction (a.k.a. conformal inference) is a user-friendly paradigm for creating statistically rigorous uncertainty sets/intervals for the predictions of such models. Critically, the sets are valid in a distribution-free sense: they possess explicit, non-asymptotic guarantees even without distributional assumptions or model assumptions.\n>\n> --- @angelopoulos2021gentle\n\nIntuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates through repeated sampling or the use of dedicated calibration data. \n\n![Conformal Prediction in action: prediction intervals at varying coverage rates. As coverage grows, so does the width of the prediction interval.](https://raw.githubusercontent.com/pat-alt/pat-alt.github.io/main/blog/posts/conformal-regression/www/medium.gif)\n\nThe animation above is lifted from a small blog [post](https://www.paltmeyer.com/blog/posts/conformal-regression/) that introduces Conformal Prediction and this package in the context of regression. It shows how the prediction interval and the test points that it covers varies in size as the user-specified coverage rate changes.\n\n## 🚩 Installation \n\nYou can install the latest stable release from the general registry:\n\n```{.julia}\nusing Pkg\nPkg.add(\"ConformalPrediction\")\n```\n\nThe development version can be installed as follows:\n\n```{.julia}\nusing Pkg\nPkg.add(url=\"https://github.com/juliatrustworthyai/ConformalPrediction.jl\")\n```\n\n## 🔍 Usage Example \n\nTo illustrate the intended use of the package, let's have a quick look at a simple regression problem. We first generate some synthetic data and then determine indices for our training and test data using [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/):\n\n::: {.cell execution_count=2}\n``` {.julia .cell-code}\nusing MLJ\n\n# Inputs:\nN = 600\nxmax = 3.0\nusing Distributions\nd = Uniform(-xmax, xmax)\nX = rand(d, N)\nX = reshape(X, :, 1)\n\n# Outputs:\nnoise = 0.5\nfun(X) = X * sin(X)\nε = randn(N) .* noise\ny = @.(fun(X)) + ε\ny = vec(y)\n\n# Partition:\ntrain, test = partition(eachindex(y), 0.4, 0.4, shuffle=true)\n```\n:::\n\n\nWe then import Symbolic Regressor ([`SymbolicRegression.jl`](https://github.com/MilesCranmer/SymbolicRegression.jl)) following the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) procedure.\n\n::: {.cell execution_count=3}\n``` {.julia .cell-code}\nregressor = @load SRRegressor pkg=SymbolicRegression\nmodel = regressor(\n niterations=50,\n binary_operators=[+, -, *],\n unary_operators=[sin],\n)\n```\n:::\n\n\nTo turn our conventional model into a conformal model, we just need to declare it as such by using `conformal_model` wrapper function. The generated conformal model instance can wrapped in data to create a *machine*. Finally, we proceed by fitting the machine on training data using the generic `fit!` method:\n\n::: {.cell execution_count=4}\n``` {.julia .cell-code}\nusing ConformalPrediction\nconf_model = conformal_model(model)\nmach = machine(conf_model, X, y)\nfit!(mach, rows=train)\n```\n:::\n\n\nPredictions can then be computed using the generic `predict` method. The code below produces predictions for the first `n` samples. Each tuple contains the lower and upper bound for the prediction interval. \n\n::: {.cell execution_count=5}\n``` {.julia .cell-code}\nshow_first = 5\nXtest = selectrows(X, test)\nytest = y[test]\nŷ = predict(mach, Xtest)\nŷ[1:show_first]\n```\n\n::: {.cell-output .cell-output-display execution_count=6}\n```\n5-element Vector{Tuple{Float64, Float64}}:\n (-0.40997718991694765, 1.449009293726001)\n (0.8484810430118421, 2.7074675266547907)\n (0.547852151594671, 2.4068386352376194)\n (-0.022697652913589494, 1.8362888307293592)\n (0.07435130847990101, 1.9333377921228496)\n```\n:::\n:::\n\n\nFor simple models like this one, we can call a custom `Plots` recipe on our instance, fit result and data to generate the chart below:\n\n::: {.cell execution_count=6}\n``` {.julia .cell-code}\nusing Plots\nzoom = 0\nplt = plot(mach.model, mach.fitresult, Xtest, ytest, lw=5, zoom=zoom, observed_lab=\"Test points\")\nxrange = range(-xmax+zoom,xmax-zoom,length=N)\nplot!(plt, xrange, @.(fun(xrange)), lw=2, ls=:dash, colour=:darkorange, label=\"Ground truth\")\n```\n\n::: {.cell-output .cell-output-display execution_count=7}\n![](index_files/figure-commonmark/cell-7-output-1.svg){}\n:::\n:::\n\n\nWe can evaluate the conformal model using the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) workflow with a custom performance measure. You can use either `emp_coverage` for the overall empirical coverage (correctness) or `ssc` for the size-stratified coverage rate (adaptiveness).\n\n::: {.cell execution_count=7}\n``` {.julia .cell-code}\n_eval = evaluate!(mach; measure=[emp_coverage, ssc], verbosity=0)\ndisplay(_eval)\nprintln(\"Empirical coverage: $(round(_eval.measurement[1], digits=3))\")\nprintln(\"SSC: $(round(_eval.measurement[2], digits=3))\")\n```\n\n::: {.cell-output .cell-output-stdout}\n```\nStarted!\n```\n:::\n\n::: {.cell-output .cell-output-display}\n```\nPerformanceEvaluation object with these fields:\n measure, operation, measurement, per_fold,\n per_observation, fitted_params_per_fold,\n report_per_fold, train_test_rows\nExtract:\n┌──────────────────────────────────────────────┬───────────┬─────────────┬──────\n│ measure │ operation │ measurement │ 1.9 ⋯\n├──────────────────────────────────────────────┼───────────┼─────────────┼──────\n│ ConformalPrediction.emp_coverage │ predict │ 0.945 │ 0.0 ⋯\n│ ConformalPrediction.size_stratified_coverage │ predict │ 0.945 │ 0.0 ⋯\n└──────────────────────────────────────────────┴───────────┴─────────────┴──────\n 2 columns omitted\n```\n:::\n:::\n\n\n## 📚 Read on\n\nIf after reading the usage example above you are just left with more questions about the topic, that's normal. Below we have have collected a number of further resources to help you get started with this package and the topic itself:\n\n1. Blog post introducing conformal classifiers: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-prediction/)], [[TDS](https://medium.com/towards-data-science/conformal-prediction-in-julia-351b81309e30)], [[Forem](https://forem.julialang.org/patalt/conformal-prediction-in-julia-h9n)].\n2. Blog post applying CP to a deep learning image classifier: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-image-classifier/)], [[TDS](https://medium.com/towards-data-science/how-to-conformalize-a-deep-image-classifier-14ead4e1a5a0)], [[Forem](https://forem.julialang.org/patalt/how-to-conformalize-a-deep-image-classifier-50p2)].\n3. The package [docs](https://juliatrustworthyai.github.io/ConformalPrediction.jl/dev/) and in particular the [FAQ](https://www.paltmeyer.com/ConformalPrediction.jl/dev/faq/).\n\n### External Resources\n\n- *A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification* by @angelopoulos2021gentle ([pdf](https://arxiv.org/pdf/2107.07511.pdf)).\n- *Predictive inference with the jackknife+* by @barber2021predictive ([pdf](https://projecteuclid.org/journals/annals-of-statistics/volume-49/issue-1/Predictive-inference-with-the-jackknife/10.1214/20-AOS1965.full))\n- *Awesome Conformal Prediction* repository by Valery Manokhin ([repo](https://github.com/valeman/awesome-conformal-prediction)).\n- [Documentation](https://mapie.readthedocs.io/en/latest/index.html) for the Python package MAPIE.\n\n## 🔁 Status \n\nThis package is in its early stages of development and therefore still subject to changes to the core architecture and API. \n\n### Implemented Methodologies\n\nThe following CP approaches have been implemented:\n\n**Regression**:\n\n- Inductive \n- Naive Transductive \n- Jackknife \n- Jackknife+ \n- Jackknife-minmax\n- CV+\n- CV-minmax\n\n**Classification**:\n\n- Inductive\n- Naive Transductive \n- Adaptive Inductive\n\nThe package has been tested for the following supervised models offered by [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/).\n\n**Regression**:\n\n::: {.cell execution_count=8}\n``` {.julia .cell-code}\nkeys(tested_atomic_models[:regression])\n```\n\n::: {.cell-output .cell-output-display execution_count=9}\n```\nKeySet for a Dict{Symbol, Expr} with 5 entries. Keys:\n :ridge\n :lasso\n :evo_tree\n :nearest_neighbor\n :linear\n```\n:::\n:::\n\n\n**Classification**:\n\n::: {.cell execution_count=9}\n``` {.julia .cell-code}\nkeys(tested_atomic_models[:classification])\n```\n\n::: {.cell-output .cell-output-display execution_count=10}\n```\nKeySet for a Dict{Symbol, Expr} with 3 entries. Keys:\n :nearest_neighbor\n :evo_tree\n :logistic\n```\n:::\n:::\n\n\n### Implemented Evaluation Metrics\n\nTo evaluate conformal predictors we are typically interested in correctness and adaptiveness. The former can be evaluated by looking at the empirical coverage rate, while the latter can be assessed through metrics that address the conditional coverage [@angelopoulos2021gentle]. To this end, the following metrics have been implemented:\n\n- `emp_coverage` (empirical coverage)\n- `ssc` (size-stratified coverage)\n\nThere is also a simple `Plots.jl` recipe that can be used to inspect the set sizes. In the regression case, the interval width is stratified into discrete bins for this purpose:\n\n::: {.cell execution_count=10}\n``` {.julia .cell-code}\nbar(mach.model, mach.fitresult, X)\n```\n\n::: {.cell-output .cell-output-display execution_count=11}\n![](index_files/figure-commonmark/cell-11-output-1.svg){}\n:::\n:::\n\n\n## 🛠 Contribute \n\nContributions are welcome! A good place to start is the [list](https://github.com/juliatrustworthyai/ConformalPrediction.jl/issues) of outstanding issues. For more details, see also the [Contributor's Guide](https://www.paltmeyer.com/ConformalPrediction.jl/dev/contribute/). Please follow the [SciML ColPrac guide](https://github.com/SciML/ColPrac).\n\n## 🙏 Thanks\n\nTo build this package I have read and re-read both @angelopoulos2021gentle and @barber2021predictive. The Awesome Conformal Prediction [repository](https://github.com/valeman/awesome-conformal-prediction) [@manokhin2022awesome] has also been a fantastic place to get started. Thanks also to [\\@aangelopoulos](https://github.com/aangelopoulos), [\\@valeman](https://github.com/valeman) and others for actively contributing to discussions on here. Quite a few people have also recently started using and contributing to the package for which I am very grateful. Finally, many thanks to Anthony Blaom ([\\@ablaom](https://github.com/ablaom)) for many helpful discussions about how to interface this package to `MLJ.jl`.\n\n## 🎓 References \n\n", + "markdown": "---\ntitle: ConformalPrediction\n---\n\n\n```@meta\nCurrentModule = ConformalPrediction\n```\n\n![](assets/wide_logo.png)\n\nDocumentation for [ConformalPrediction.jl](https://github.com/juliatrustworthyai/ConformalPrediction.jl).\n\n\n\n\n\n`ConformalPrediction.jl` is a package for Predictive Uncertainty Quantification (UQ) through Conformal Prediction (CP) in Julia. It is designed to work with supervised models trained in [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) [@blaom2020mlj]. Conformal Prediction is easy-to-understand, easy-to-use and model-agnostic and it works under minimal distributional assumptions. \n\n## 🏃 Quick Tour\n\n> First time here? Take a quick interactive [tour](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) to see what this package can do right on [JuliaHub](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl){target=\"_blank\"} (To run the notebook, hit login and then edit). \n\nThis [`Pluto.jl`](https://github.com/fonsp/Pluto.jl){target=\"_blank\"} 🎈 notebook won the 2nd Price in the [JuliaCon 2023 Notebook Competition](https://info.juliahub.com/pluto-notebook-winner-23).\n\n### Local Tour\n\nTo run the tour locally, just clone this repo and start `Pluto.jl` as follows:\n\n```{.julia}\n] add Pluto\nusing Pluto\nPluto.run()\n```\n\nAll notebooks are contained in `docs/pluto`.\n\n## 📖 Background\n\nDon't worry, we're not about to deep-dive into methodology. But just to give you a high-level description of Conformal Prediction (CP) upfront:\n\n> Conformal prediction (a.k.a. conformal inference) is a user-friendly paradigm for creating statistically rigorous uncertainty sets/intervals for the predictions of such models. Critically, the sets are valid in a distribution-free sense: they possess explicit, non-asymptotic guarantees even without distributional assumptions or model assumptions.\n>\n> --- @angelopoulos2021gentle\n\nIntuitively, CP works under the premise of turning heuristic notions of uncertainty into rigorous uncertainty estimates through repeated sampling or the use of dedicated calibration data. \n\n![Conformal Prediction in action: prediction intervals at varying coverage rates. As coverage grows, so does the width of the prediction interval.](https://raw.githubusercontent.com/pat-alt/pat-alt.github.io/main/blog/posts/conformal-regression/www/medium.gif)\n\nThe animation above is lifted from a small blog [post](https://www.paltmeyer.com/blog/posts/conformal-regression/) that introduces Conformal Prediction and this package in the context of regression. It shows how the prediction interval and the test points that it covers varies in size as the user-specified coverage rate changes.\n\n## 🚩 Installation \n\nYou can install the latest stable release from the general registry:\n\n```{.julia}\nusing Pkg\nPkg.add(\"ConformalPrediction\")\n```\n\nThe development version can be installed as follows:\n\n```{.julia}\nusing Pkg\nPkg.add(url=\"https://github.com/juliatrustworthyai/ConformalPrediction.jl\")\n```\n\n## 🔍 Usage Example \n\nTo illustrate the intended use of the package, let's have a quick look at a simple regression problem. We first generate some synthetic data and then determine indices for our training and test data using [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/):\n\n::: {.cell execution_count=2}\n``` {.julia .cell-code}\nusing MLJ\n\n# Inputs:\nN = 600\nxmax = 3.0\nusing Distributions\nd = Uniform(-xmax, xmax)\nX = rand(d, N)\nX = reshape(X, :, 1)\n\n# Outputs:\nnoise = 0.5\nfun(X) = sin(X)\nε = randn(N) .* noise\ny = @.(fun(X)) + ε\ny = vec(y)\n\n# Partition:\ntrain, test = partition(eachindex(y), 0.4, 0.4, shuffle=true)\n```\n:::\n\n\nWe then import Symbolic Regressor ([`SymbolicRegression.jl`](https://github.com/MilesCranmer/SymbolicRegression.jl)) following the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) procedure.\n\n::: {.cell execution_count=3}\n``` {.julia .cell-code}\nregressor = @load SRRegressor pkg=SymbolicRegression\nmodel = regressor(\n niterations=50,\n binary_operators=[+, -, *],\n unary_operators=[sin],\n)\n```\n:::\n\n\nTo turn our conventional model into a conformal model, we just need to declare it as such by using `conformal_model` wrapper function. The generated conformal model instance can wrapped in data to create a *machine*. Finally, we proceed by fitting the machine on training data using the generic `fit!` method:\n\n::: {.cell execution_count=4}\n``` {.julia .cell-code}\nusing ConformalPrediction\nconf_model = conformal_model(model)\nmach = machine(conf_model, X, y)\nfit!(mach, rows=train)\n```\n:::\n\n\nPredictions can then be computed using the generic `predict` method. The code below produces predictions for the first `n` samples. Each tuple contains the lower and upper bound for the prediction interval. \n\n::: {.cell execution_count=5}\n``` {.julia .cell-code}\nshow_first = 5\nXtest = selectrows(X, test)\nytest = y[test]\nŷ = predict(mach, Xtest)\nŷ[1:show_first]\n```\n\n::: {.cell-output .cell-output-display execution_count=6}\n```\n5-element Vector{Tuple{Float64, Float64}}:\n (-0.04087262272113379, 1.8635644669554758)\n (0.04647464096907805, 1.9509117306456876)\n (-0.24248802236397216, 1.6619490673126376)\n (-0.07841928163933476, 1.8260178080372749)\n (-0.02268628324126465, 1.881750806435345)\n```\n:::\n:::\n\n\nFor simple models like this one, we can call a custom `Plots` recipe on our instance, fit result and data to generate the chart below:\n\n::: {.cell execution_count=6}\n``` {.julia .cell-code}\nusing Plots\nzoom = 0\nplt = plot(mach.model, mach.fitresult, Xtest, ytest, lw=5, zoom=zoom, observed_lab=\"Test points\")\nxrange = range(-xmax+zoom,xmax-zoom,length=N)\nplot!(plt, xrange, @.(fun(xrange)), lw=2, ls=:dash, colour=:darkorange, label=\"Ground truth\")\n```\n\n::: {.cell-output .cell-output-display execution_count=7}\n![](index_files/figure-commonmark/cell-7-output-1.svg){}\n:::\n:::\n\n\n\n\nWe can evaluate the conformal model using the standard [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) workflow with a custom performance measure. You can use either `emp_coverage` for the overall empirical coverage (correctness) or `ssc` for the size-stratified coverage rate (adaptiveness).\n\n::: {.cell execution_count=8}\n``` {.julia .cell-code}\n_eval = evaluate!(mach; measure=[emp_coverage, ssc], verbosity=0)\ndisplay(_eval)\nprintln(\"Empirical coverage: $(round(_eval.measurement[1], digits=3))\")\nprintln(\"SSC: $(round(_eval.measurement[2], digits=3))\")\n```\n\n::: {.cell-output .cell-output-display}\n```\nPerformanceEvaluation object with these fields:\n model, measure, operation, measurement, per_fold,\n per_observation, fitted_params_per_fold,\n report_per_fold, train_test_rows, resampling, repeats\nExtract:\n┌──────────────────────────────────────────────┬───────────┬─────────────┬──────\n│ measure │ operation │ measurement │ 1.9 ⋯\n├──────────────────────────────────────────────┼───────────┼─────────────┼──────\n│ ConformalPrediction.emp_coverage │ predict │ 0.953 │ 0.0 ⋯\n│ ConformalPrediction.size_stratified_coverage │ predict │ 0.953 │ 0.0 ⋯\n└──────────────────────────────────────────────┴───────────┴─────────────┴──────\n 2 columns omitted\n```\n:::\n\n::: {.cell-output .cell-output-stdout}\n```\nEmpirical coverage: 0.953\nSSC: 0.953\n```\n:::\n:::\n\n\n## 📚 Read on\n\nIf after reading the usage example above you are just left with more questions about the topic, that's normal. Below we have have collected a number of further resources to help you get started with this package and the topic itself:\n\n1. Blog post introducing conformal classifiers: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-prediction/)], [[TDS](https://medium.com/towards-data-science/conformal-prediction-in-julia-351b81309e30)], [[Forem](https://forem.julialang.org/patalt/conformal-prediction-in-julia-h9n)].\n2. Blog post applying CP to a deep learning image classifier: [[Quarto](https://www.paltmeyer.com/blog/posts/conformal-image-classifier/)], [[TDS](https://medium.com/towards-data-science/how-to-conformalize-a-deep-image-classifier-14ead4e1a5a0)], [[Forem](https://forem.julialang.org/patalt/how-to-conformalize-a-deep-image-classifier-50p2)].\n3. The package [docs](https://juliatrustworthyai.github.io/ConformalPrediction.jl/dev/) and in particular the [FAQ](https://www.paltmeyer.com/ConformalPrediction.jl/dev/faq/).\n\n### External Resources\n\n- *A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification* by @angelopoulos2021gentle ([pdf](https://arxiv.org/pdf/2107.07511.pdf)).\n- *Predictive inference with the jackknife+* by @barber2021predictive ([pdf](https://projecteuclid.org/journals/annals-of-statistics/volume-49/issue-1/Predictive-inference-with-the-jackknife/10.1214/20-AOS1965.full))\n- *Awesome Conformal Prediction* repository by Valery Manokhin ([repo](https://github.com/valeman/awesome-conformal-prediction)).\n- [Documentation](https://mapie.readthedocs.io/en/latest/index.html) for the Python package MAPIE.\n\n## 🔁 Status \n\nThis package is in its early stages of development and therefore still subject to changes to the core architecture and API. \n\n### Implemented Methodologies\n\nThe following CP approaches have been implemented:\n\n**Regression**:\n\n- Inductive \n- Naive Transductive \n- Jackknife \n- Jackknife+ \n- Jackknife-minmax\n- CV+\n- CV-minmax\n\n**Classification**:\n\n- Inductive\n- Naive Transductive \n- Adaptive Inductive\n\nThe package has been tested for the following supervised models offered by [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/).\n\n**Regression**:\n\n::: {.cell execution_count=9}\n``` {.julia .cell-code}\nkeys(tested_atomic_models[:regression])\n```\n\n::: {.cell-output .cell-output-display execution_count=9}\n```\nKeySet for a Dict{Symbol, Expr} with 5 entries. Keys:\n :ridge\n :lasso\n :evo_tree\n :nearest_neighbor\n :linear\n```\n:::\n:::\n\n\n**Classification**:\n\n::: {.cell execution_count=10}\n``` {.julia .cell-code}\nkeys(tested_atomic_models[:classification])\n```\n\n::: {.cell-output .cell-output-display execution_count=10}\n```\nKeySet for a Dict{Symbol, Expr} with 3 entries. Keys:\n :nearest_neighbor\n :evo_tree\n :logistic\n```\n:::\n:::\n\n\n### Implemented Evaluation Metrics\n\nTo evaluate conformal predictors we are typically interested in correctness and adaptiveness. The former can be evaluated by looking at the empirical coverage rate, while the latter can be assessed through metrics that address the conditional coverage [@angelopoulos2021gentle]. To this end, the following metrics have been implemented:\n\n- `emp_coverage` (empirical coverage)\n- `ssc` (size-stratified coverage)\n\nThere is also a simple `Plots.jl` recipe that can be used to inspect the set sizes. In the regression case, the interval width is stratified into discrete bins for this purpose:\n\n::: {.cell execution_count=11}\n``` {.julia .cell-code}\nbar(mach.model, mach.fitresult, X)\n```\n\n::: {.cell-output .cell-output-display execution_count=11}\n![](index_files/figure-commonmark/cell-12-output-1.svg){}\n:::\n:::\n\n\n## 🛠 Contribute \n\nContributions are welcome! A good place to start is the [list](https://github.com/juliatrustworthyai/ConformalPrediction.jl/issues) of outstanding issues. For more details, see also the [Contributor's Guide](https://www.paltmeyer.com/ConformalPrediction.jl/dev/contribute/). Please follow the [SciML ColPrac guide](https://github.com/SciML/ColPrac).\n\n## 🙏 Thanks\n\nTo build this package I have read and re-read both @angelopoulos2021gentle and @barber2021predictive. The Awesome Conformal Prediction [repository](https://github.com/valeman/awesome-conformal-prediction) [@manokhin2022awesome] has also been a fantastic place to get started. Thanks also to [\\@aangelopoulos](https://github.com/aangelopoulos), [\\@valeman](https://github.com/valeman) and others for actively contributing to discussions on here. Quite a few people have also recently started using and contributing to the package for which I am very grateful. Finally, many thanks to Anthony Blaom ([\\@ablaom](https://github.com/ablaom)) for many helpful discussions about how to interface this package to `MLJ.jl`.\n\n## 🎓 References \n\n", "supporting": [ "index_files/figure-commonmark" ], diff --git a/_freeze/docs/src/index/figure-commonmark/cell-12-output-1.svg b/_freeze/docs/src/index/figure-commonmark/cell-12-output-1.svg new file mode 100644 index 0000000..c459847 --- /dev/null +++ b/_freeze/docs/src/index/figure-commonmark/cell-12-output-1.svg @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/_freeze/docs/src/index/figure-commonmark/cell-7-output-1.svg b/_freeze/docs/src/index/figure-commonmark/cell-7-output-1.svg index ea41553..5463e11 100644 --- a/_freeze/docs/src/index/figure-commonmark/cell-7-output-1.svg +++ b/_freeze/docs/src/index/figure-commonmark/cell-7-output-1.svg @@ -1,294 +1,285 @@ - + - + - + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/index.md b/docs/src/index.md index b37fdc1..9a4b7ce 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -12,9 +12,9 @@ Documentation for [ConformalPrediction.jl](https://github.com/juliatrustworthyai ## 🏃 Quick Tour -> First time here? Take a quick interactive [tour](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl) to see what this package can do: [![Binder](https://mybinder.org/badge_logo.svg)](https://binder.plutojl.org/v0.19.12/open?url=https%253A%252F%252Fraw.githubusercontent.com%252Fpat-alt%252FConformalPrediction.jl%252Fmain%252Fdocs%252Fpluto%252Fintro.jl) +> First time here? Take a quick interactive [tour](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) to see what this package can do right on [JuliaHub](https://juliahub.com/ui/Notebooks/juliahub/Tutorials/ConformalPrediction.jl) (To run the notebook, hit login and then edit). -The button takes you to a [`Pluto.jl`](https://github.com/fonsp/Pluto.jl) 🎈 notebook hosted on [binder](https://mybinder.org/). In my own experience, this may take some time to load, certainly long enough to get yourself a hot beverage ☕. Alternatively, you can run the notebook locally or skip the tour for now and read on below. +This [`Pluto.jl`](https://github.com/fonsp/Pluto.jl) 🎈 notebook won the 2nd Price in the [JuliaCon 2023 Notebook Competition](https://info.juliahub.com/pluto-notebook-winner-23). ### Local Tour @@ -75,7 +75,7 @@ X = reshape(X, :, 1) # Outputs: noise = 0.5 -fun(X) = X * sin(X) +fun(X) = sin(X) ε = randn(N) .* noise y = @.(fun(X)) + ε y = vec(y) @@ -115,11 +115,11 @@ ŷ[1:show_first] ``` 5-element Vector{Tuple{Float64, Float64}}: - (-0.40997718991694765, 1.449009293726001) - (0.8484810430118421, 2.7074675266547907) - (0.547852151594671, 2.4068386352376194) - (-0.022697652913589494, 1.8362888307293592) - (0.07435130847990101, 1.9333377921228496) + (-0.04087262272113379, 1.8635644669554758) + (0.04647464096907805, 1.9509117306456876) + (-0.24248802236397216, 1.6619490673126376) + (-0.07841928163933476, 1.8260178080372749) + (-0.02268628324126465, 1.881750806435345) For simple models like this one, we can call a custom `Plots` recipe on our instance, fit result and data to generate the chart below: @@ -142,21 +142,22 @@ println("Empirical coverage: $(round(_eval.measurement[1], digits=3))") println("SSC: $(round(_eval.measurement[2], digits=3))") ``` - Started! - PerformanceEvaluation object with these fields: - measure, operation, measurement, per_fold, + model, measure, operation, measurement, per_fold, per_observation, fitted_params_per_fold, - report_per_fold, train_test_rows + report_per_fold, train_test_rows, resampling, repeats Extract: ┌──────────────────────────────────────────────┬───────────┬─────────────┬────── │ measure │ operation │ measurement │ 1.9 ⋯ ├──────────────────────────────────────────────┼───────────┼─────────────┼────── - │ ConformalPrediction.emp_coverage │ predict │ 0.945 │ 0.0 ⋯ - │ ConformalPrediction.size_stratified_coverage │ predict │ 0.945 │ 0.0 ⋯ + │ ConformalPrediction.emp_coverage │ predict │ 0.953 │ 0.0 ⋯ + │ ConformalPrediction.size_stratified_coverage │ predict │ 0.953 │ 0.0 ⋯ └──────────────────────────────────────────────┴───────────┴─────────────┴────── 2 columns omitted + Empirical coverage: 0.953 + SSC: 0.953 + ## 📚 Read on If after reading the usage example above you are just left with more questions about the topic, that’s normal. Below we have have collected a number of further resources to help you get started with this package and the topic itself: @@ -235,7 +236,7 @@ There is also a simple `Plots.jl` recipe that can be used to inspect the set siz bar(mach.model, mach.fitresult, X) ``` -![](index_files/figure-commonmark/cell-11-output-1.svg) +![](index_files/figure-commonmark/cell-12-output-1.svg) ## 🛠 Contribute diff --git a/docs/src/index_files/figure-commonmark/cell-12-output-1.svg b/docs/src/index_files/figure-commonmark/cell-12-output-1.svg new file mode 100644 index 0000000..c459847 --- /dev/null +++ b/docs/src/index_files/figure-commonmark/cell-12-output-1.svg @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/index_files/figure-commonmark/cell-7-output-1.svg b/docs/src/index_files/figure-commonmark/cell-7-output-1.svg index ea41553..5463e11 100644 --- a/docs/src/index_files/figure-commonmark/cell-7-output-1.svg +++ b/docs/src/index_files/figure-commonmark/cell-7-output-1.svg @@ -1,294 +1,285 @@ - + - + - + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +