From 2e0be2e62792c3f338cd809fbe05aa083dbee329 Mon Sep 17 00:00:00 2001 From: pat-alt Date: Thu, 6 Jul 2023 15:43:42 +0200 Subject: [PATCH 1/3] added --- .github/workflows/FormatCheck.yml | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/FormatCheck.yml diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml new file mode 100644 index 0000000..e576a1b --- /dev/null +++ b/.github/workflows/FormatCheck.yml @@ -0,0 +1,34 @@ +name: Format Check + +on: + push: + branches: + - 'main' + - 'release-' + tags: '*' + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: julia-actions/setup-julia@latest + with: + version: 1 + - uses: actions/checkout@v1 + - name: Install JuliaFormatter + run: | + using Pkg + Pkg.add("JuliaFormatter") + shell: julia --color=yes {0} + - name: Format code + run: | + using JuliaFormatter + format("."; verbose=true) + shell: julia --color=yes {0} + - name: Suggest formatting changes + uses: reviewdog/action-suggester@v1 + if: github.event_name == 'pull_request' + with: + tool_name: JuliaFormatter + fail_on_error: true \ No newline at end of file From cc28b410b93594b7bf2341cf726b8396f65303bb Mon Sep 17 00:00:00 2001 From: pat-alt Date: Thu, 6 Jul 2023 15:51:31 +0200 Subject: [PATCH 2/3] now corrected format --- docs/make.jl | 4 +- docs/pluto/intro.jl | 175 ++++++++------- docs/pluto/intro_dev.jl | 200 +++++++++--------- docs/pluto/understanding_coverage.jl | 127 ++++++----- docs/setup_docs.jl | 2 +- src/ConformalPrediction.jl | 2 +- src/artifacts/core.jl | 2 +- src/conformal_models/conformal_models.jl | 7 +- .../inductive_classification.jl | 45 ++-- src/conformal_models/plotting.jl | 27 ++- .../training/inductive_classification.jl | 52 +++-- src/conformal_models/training/losses.jl | 49 +++-- src/conformal_models/training/training.jl | 2 +- .../transductive_classification.jl | 2 +- .../transductive_regression.jl | 126 +++++++---- 15 files changed, 477 insertions(+), 345 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 869b4ee..edcf871 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -54,6 +54,6 @@ makedocs(; ) deploydocs(; - repo = "github.com/JuliaTrustworthyAI/ConformalPrediction.jl", - devbranch = "main" + repo = "github.com/JuliaTrustworthyAI/ConformalPrediction.jl", + devbranch = "main", ) diff --git a/docs/pluto/intro.jl b/docs/pluto/intro.jl index edfffac..08ebd3e 100644 --- a/docs/pluto/intro.jl +++ b/docs/pluto/intro.jl @@ -7,7 +7,14 @@ using InteractiveUtils # This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error). macro bind(def, element) quote - local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end + local iv = try + Base.loaded_modules[Base.PkgId( + Base.UUID("6e696c72-6542-2067-7265-42206c756150"), + "AbstractPlutoDingetjes", + )].Bonds.initial_value + catch + b -> missing + end local el = $(esc(element)) global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el) el @@ -22,7 +29,7 @@ begin using EvoTrees: EvoTreeRegressor using LightGBM.MLJInterface: LGBMRegressor using MLJBase - using MLJLinearModels + using MLJLinearModels using MLJModels using NearestNeighborModels: KNNRegressor using Plots @@ -44,25 +51,25 @@ Let's start by loading the necessary packages: # ╔═╡ 55a7c16b-a526-41d9-9d73-a0591ad006ce # helper functions begin - function multi_slider(vals::Dict; title = "") - - return PlutoUI.combine() do Child - - inputs = [ - md""" $(_name): $( - Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) - )""" - - for (_name, _vals) in vals - ] - - md""" - #### $title - $(inputs) - """ - end - - end + function multi_slider(vals::Dict; title = "") + + return PlutoUI.combine() do Child + + inputs = [ + md""" $(_name): $( + Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) + )""" + + for (_name, _vals) in vals + ] + + md""" + #### $title + $(inputs) + """ + end + + end end; # ╔═╡ be8b2fbb-3b3d-496e-9041-9b8f50872350 @@ -89,7 +96,7 @@ Most machine learning workflows start with data. In this tutorial you have full # ╔═╡ 2f1c8da3-77dc-4bd7-8fa4-7669c2861aaa begin - function get_data(N=600, xmax=3.0, noise=0.5; fun::Function=fun(X) = X * sin(X)) + function get_data(N = 600, xmax = 3.0, noise = 0.5; fun::Function = fun(X) = X * sin(X)) # Inputs: d = Distributions.Uniform(-xmax, xmax) X = rand(d, N) @@ -123,21 +130,21 @@ begin "noise" => (0.1:0.1:1.0, 0.5), "xmax" => (1:10, 5), ) - @bind data_specs multi_slider(data_dict, title="Parameters") + @bind data_specs multi_slider(data_dict, title = "Parameters") end # ╔═╡ f0106aa5-b1c5-4857-af94-2711f80d25a8 begin - X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun=f) - scatter(X.x1, y, label="Observed data") - xrange = range(-data_specs.xmax, data_specs.xmax, length=50) + X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun = f) + scatter(X.x1, y, label = "Observed data") + xrange = range(-data_specs.xmax, data_specs.xmax, length = 50) plot!( xrange, @.(f(xrange)), - lw=4, - label="Ground truth", - ls=:dash, - colour=:black, + lw = 4, + label = "Ground truth", + ls = :dash, + colour = :black, ) end @@ -158,7 +165,7 @@ To start with, let's split our data into a training and test set: """ # ╔═╡ 3a4fe2bc-387c-4d7e-b45f-292075a01bcd -train, test = partition(eachindex(y), 0.4, 0.4, shuffle=true); +train, test = partition(eachindex(y), 0.4, 0.4, shuffle = true); # ╔═╡ a34b8c07-08e0-4a0e-a0f9-8054b41b038b md"Now let's choose a model for our regression task:" @@ -184,7 +191,7 @@ mach_raw = machine(model, X, y); md"Then we fit the machine to the training data:" # ╔═╡ aabfbbfb-7fb0-4f37-9a05-b96207636232 -MLJBase.fit!(mach_raw, rows=train, verbosity=0); +MLJBase.fit!(mach_raw, rows = train, verbosity = 0); # ╔═╡ 5506e1b5-5f2f-4972-a845-9c0434d4b31c md""" @@ -196,16 +203,16 @@ begin Xtest = MLJBase.matrix(selectrows(X, test)) ytest = y[test] ŷ = MLJBase.predict(mach_raw, Xtest) - scatter(vec(Xtest), vec(ytest), label="Observed") + scatter(vec(Xtest), vec(ytest), label = "Observed") _order = sortperm(vec(Xtest)) - plot!(vec(Xtest)[_order], vec(ŷ)[_order], lw=4, label="Predicted") + plot!(vec(Xtest)[_order], vec(ŷ)[_order], lw = 4, label = "Predicted") plot!( xrange, @.(f(xrange)), - lw=2, - ls=:dash, - colour=:black, - label="Ground truth", + lw = 2, + ls = :dash, + colour = :black, + label = "Ground truth", ) end @@ -240,7 +247,7 @@ Then we fit the machine to the data: """ # ╔═╡ 6b574688-ff3c-441a-a616-169685731883 -MLJBase.fit!(mach, rows=train, verbosity=0); +MLJBase.fit!(mach, rows = train, verbosity = 0); # ╔═╡ da6e8f90-a3f9-4d06-86ab-b0f6705bbf54 md""" @@ -250,21 +257,21 @@ Now let us look at the predictions for our test data again. The chart below show """ # ╔═╡ 797746e9-235f-4fb1-8cdb-9be295b54bbe -@bind coverage Slider(0.1:0.1:1.0, default=0.8, show_value=true) +@bind coverage Slider(0.1:0.1:1.0, default = 0.8, show_value = true) # ╔═╡ ad3e290b-c1f5-4008-81c7-a1a56ab10563 begin - _conf_model = conformal_model(model, coverage=coverage) + _conf_model = conformal_model(model, coverage = coverage) _mach = machine(_conf_model, X, y) - MLJBase.fit!(_mach, rows=train, verbosity=0) - plot(_mach.model, _mach.fitresult, Xtest, ytest, zoom=0, observed_lab="Test points") + MLJBase.fit!(_mach, rows = train, verbosity = 0) + plot(_mach.model, _mach.fitresult, Xtest, ytest, zoom = 0, observed_lab = "Test points") plot!( xrange, @.(f(xrange)), - lw=2, - ls=:dash, - colour=:black, - label="Ground truth", + lw = 2, + ls = :dash, + colour = :black, + label = "Ground truth", ) end @@ -287,7 +294,7 @@ To verify the marginal coverage property empirically we can look at the empirica # ╔═╡ d1140af9-608a-4669-9595-aee72ffbaa46 begin model_evaluation = - evaluate!(_mach, operation=MLJBase.predict, measure=emp_coverage, verbosity=0) + evaluate!(_mach, operation = MLJBase.predict, measure = emp_coverage, verbosity = 0) println("Empirical coverage: $(round(model_evaluation.measurement[1], digits=3))") println("Coverage per fold: $(round.(model_evaluation.per_fold[1], digits=3))") end @@ -295,23 +302,23 @@ end # ╔═╡ f742440b-258e-488a-9c8b-c9267cf1fb99 begin ncal = Int(conf_model.train_ratio * data_specs.N) - if model_evaluation.measurement[1] < coverage - Markdown.parse( - """ - > ❌❌❌ Oh no! You got an empirical coverage rate that is slightly lower than desired 🥲 ... what's happened? - - The coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). - """ - ) - else - Markdown.parse( - """ - > ✅ ✅ ✅ Great! You got an empirical coverage rate that is slightly higher than desired 😁 ... but why isn't it exactly the same? - - In most cases it will be slightly higher than desired, since ``(1-\\alpha)`` is a lower bound. But note that it can also be slightly lower than desired. That is because the coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). - """ - ) - end + if model_evaluation.measurement[1] < coverage + Markdown.parse( + """ + > ❌❌❌ Oh no! You got an empirical coverage rate that is slightly lower than desired 🥲 ... what's happened? + + The coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). + """, + ) + else + Markdown.parse( + """ + > ✅ ✅ ✅ Great! You got an empirical coverage rate that is slightly higher than desired 😁 ... but why isn't it exactly the same? + + In most cases it will be slightly higher than desired, since ``(1-\\alpha)`` is a lower bound. But note that it can also be slightly lower than desired. That is because the coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). + """, + ) + end end # ╔═╡ f7b2296f-919f-4870-aac1-8e36dd694422 @@ -343,36 +350,40 @@ Quite cool, right? Using a single API call we are able to generate rigorous pred # ╔═╡ 824bd383-2fcb-4888-8ad1-260c85333edf @bind xmax_ood Slider( data_specs.xmax:(data_specs.xmax+5), - default=(data_specs.xmax), - show_value=true, + default = (data_specs.xmax), + show_value = true, ) # ╔═╡ 072cc72d-20a2-4ee9-954c-7ea70dfb8eea begin - Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun=f) - plot(_mach.model, _mach.fitresult, Xood, yood, zoom=0, observed_lab="Test points") - xood_range = range(-xmax_ood, xmax_ood, length=50) + Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun = f) + plot(_mach.model, _mach.fitresult, Xood, yood, zoom = 0, observed_lab = "Test points") + xood_range = range(-xmax_ood, xmax_ood, length = 50) plot!( xood_range, @.(f(xood_range)), - lw=2, - ls=:dash, - colour=:black, - label="Ground truth", + lw = 2, + ls = :dash, + colour = :black, + label = "Ground truth", ) end # ╔═╡ 4f41ec7c-aedd-475f-942d-33e2d1174902 if xmax_ood > data_specs.xmax - Markdown.parse(""" - > Whooooops 🤕 ... looks like we're in trouble! What happened here? - - By expaning the domain of out inputs, we have violated the exchangeability assumption. When that assumption is violated, the marginal coverage property does not hold. But do not despair! There are ways to deal with this. - """) + Markdown.parse( + """ +> Whooooops 🤕 ... looks like we're in trouble! What happened here? + +By expaning the domain of out inputs, we have violated the exchangeability assumption. When that assumption is violated, the marginal coverage property does not hold. But do not despair! There are ways to deal with this. +""", + ) else - Markdown.parse(""" - > Still looking OK 🤨 ... Try moving the slider above the chart to the right to see what will happen. - """) + Markdown.parse( + """ +> Still looking OK 🤨 ... Try moving the slider above the chart to the right to see what will happen. +""", + ) end # ╔═╡ c7fa1889-b0be-4d96-b845-e79fa7932b0c diff --git a/docs/pluto/intro_dev.jl b/docs/pluto/intro_dev.jl index 45a3076..c991a44 100644 --- a/docs/pluto/intro_dev.jl +++ b/docs/pluto/intro_dev.jl @@ -7,7 +7,14 @@ using InteractiveUtils # This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error). macro bind(def, element) quote - local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end + local iv = try + Base.loaded_modules[Base.PkgId( + Base.UUID("6e696c72-6542-2067-7265-42206c756150"), + "AbstractPlutoDingetjes", + )].Bonds.initial_value + catch + b -> missing + end local el = $(esc(element)) global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el) el @@ -16,17 +23,18 @@ end # ╔═╡ aad62ef1-4136-4732-a9e6-3746524978ee begin - using Pkg; Pkg.activate("../") + using Pkg + Pkg.activate("../") using ConformalPrediction using DecisionTree: DecisionTreeRegressor using Distributions using EvoTrees: EvoTreeRegressor - using Flux + using Flux using LightGBM.MLJInterface: LGBMRegressor using MLJBase - using MLJFlux - using MLJFlux: NeuralNetworkRegressor - using MLJLinearModels + using MLJFlux + using MLJFlux: NeuralNetworkRegressor + using MLJLinearModels using MLJModels using NearestNeighborModels: KNNRegressor using Plots @@ -45,27 +53,27 @@ Let's start by loading the necessary packages: # ╔═╡ 55a7c16b-a526-41d9-9d73-a0591ad006ce # helper functions begin - function multi_slider(vals::Dict; title = "") - - return PlutoUI.combine() do Child - - inputs = [ - md""" $(_name): $( - Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) - )""" - - for (_name, _vals) in vals - ] - - md""" - #### $title - $(inputs) - """ - end - - end - - MLJFlux.reformat(X, ::Type{<:AbstractMatrix}) = X' + function multi_slider(vals::Dict; title = "") + + return PlutoUI.combine() do Child + + inputs = [ + md""" $(_name): $( + Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) + )""" + + for (_name, _vals) in vals + ] + + md""" + #### $title + $(inputs) + """ + end + + end + + MLJFlux.reformat(X, ::Type{<:AbstractMatrix}) = X' end; # ╔═╡ be8b2fbb-3b3d-496e-9041-9b8f50872350 @@ -92,7 +100,7 @@ Most machine learning workflows start with data. In this tutorial you have full # ╔═╡ 2f1c8da3-77dc-4bd7-8fa4-7669c2861aaa begin - function get_data(N=600, xmax=3.0, noise=0.5; fun::Function=fun(X) = X * sin(X)) + function get_data(N = 600, xmax = 3.0, noise = 0.5; fun::Function = fun(X) = X * sin(X)) # Inputs: d = Distributions.Uniform(-xmax, xmax) X = Float32.(rand(d, N)) @@ -126,21 +134,21 @@ begin "noise" => (0.1:0.1:1.0, 0.5), "xmax" => (1:10, 5), ) - @bind data_specs multi_slider(data_dict, title="Parameters") + @bind data_specs multi_slider(data_dict, title = "Parameters") end # ╔═╡ f0106aa5-b1c5-4857-af94-2711f80d25a8 begin - X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun=f) - scatter(X.x1, y, label="Observed data") - xrange = range(-data_specs.xmax, data_specs.xmax, length=50) + X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun = f) + scatter(X.x1, y, label = "Observed data") + xrange = range(-data_specs.xmax, data_specs.xmax, length = 50) plot!( xrange, @.(f(xrange)), - lw=4, - label="Ground truth", - ls=:dash, - colour=:black, + lw = 4, + label = "Ground truth", + ls = :dash, + colour = :black, ) end @@ -161,7 +169,7 @@ To start with, let's split our data into a training and test set: """ # ╔═╡ 3a4fe2bc-387c-4d7e-b45f-292075a01bcd -train, test = partition(eachindex(y), 0.4, 0.4, shuffle=true); +train, test = partition(eachindex(y), 0.4, 0.4, shuffle = true); # ╔═╡ a34b8c07-08e0-4a0e-a0f9-8054b41b038b md"Now let's choose a model for our regression task:" @@ -173,13 +181,11 @@ md"Now let's choose a model for our regression task:" begin Model = eval(tested_atomic_models[:regression][model_name]) if Model() isa MLJFlux.MLJFluxModel - model = Model( - builder=MLJFlux.MLP(hidden=(50,), σ=Flux.tanh_fast), - epochs=200 - ) - else - model = Model() - end + model = + Model(builder = MLJFlux.MLP(hidden = (50,), σ = Flux.tanh_fast), epochs = 200) + else + model = Model() + end end; # ╔═╡ 10340f3f-7981-42da-846a-7599a9edb7f3 @@ -194,7 +200,7 @@ mach_raw = machine(model, X, y); md"Then we fit the machine to the training data:" # ╔═╡ aabfbbfb-7fb0-4f37-9a05-b96207636232 -MLJBase.fit!(mach_raw, rows=train, verbosity=0); +MLJBase.fit!(mach_raw, rows = train, verbosity = 0); # ╔═╡ 5506e1b5-5f2f-4972-a845-9c0434d4b31c md""" @@ -206,16 +212,16 @@ begin Xtest = MLJBase.matrix(selectrows(X, test)) ytest = y[test] ŷ = MLJBase.predict(mach_raw, Xtest) - scatter(vec(Xtest), vec(ytest), label="Observed") + scatter(vec(Xtest), vec(ytest), label = "Observed") _order = sortperm(vec(Xtest)) - plot!(vec(Xtest)[_order], vec(ŷ)[_order], lw=4, label="Predicted") + plot!(vec(Xtest)[_order], vec(ŷ)[_order], lw = 4, label = "Predicted") plot!( xrange, @.(f(xrange)), - lw=2, - ls=:dash, - colour=:black, - label="Ground truth", + lw = 2, + ls = :dash, + colour = :black, + label = "Ground truth", ) end @@ -250,7 +256,7 @@ Then we fit the machine to the data: """ # ╔═╡ 6b574688-ff3c-441a-a616-169685731883 -MLJBase.fit!(mach, rows=train, verbosity=0); +MLJBase.fit!(mach, rows = train, verbosity = 0); # ╔═╡ da6e8f90-a3f9-4d06-86ab-b0f6705bbf54 md""" @@ -260,21 +266,21 @@ Now let us look at the predictions for our test data again. The chart below show """ # ╔═╡ 797746e9-235f-4fb1-8cdb-9be295b54bbe -@bind coverage Slider(0.1:0.1:1.0, default=0.8, show_value=true) +@bind coverage Slider(0.1:0.1:1.0, default = 0.8, show_value = true) # ╔═╡ ad3e290b-c1f5-4008-81c7-a1a56ab10563 begin - _conf_model = conformal_model(model, coverage=coverage) + _conf_model = conformal_model(model, coverage = coverage) _mach = machine(_conf_model, X, y) - MLJBase.fit!(_mach, rows=train, verbosity=0) - plot(_mach.model, _mach.fitresult, Xtest, ytest, zoom=0, observed_lab="Test points") + MLJBase.fit!(_mach, rows = train, verbosity = 0) + plot(_mach.model, _mach.fitresult, Xtest, ytest, zoom = 0, observed_lab = "Test points") plot!( xrange, @.(f(xrange)), - lw=2, - ls=:dash, - colour=:black, - label="Ground truth", + lw = 2, + ls = :dash, + colour = :black, + label = "Ground truth", ) end @@ -297,7 +303,7 @@ To verify the marginal coverage property empirically we can look at the empirica # ╔═╡ d1140af9-608a-4669-9595-aee72ffbaa46 begin model_evaluation = - evaluate!(_mach, operation=MLJBase.predict, measure=emp_coverage, verbosity=0) + evaluate!(_mach, operation = MLJBase.predict, measure = emp_coverage, verbosity = 0) println("Empirical coverage: $(round(model_evaluation.measurement[1], digits=3))") println("Coverage per fold: $(round.(model_evaluation.per_fold[1], digits=3))") end @@ -305,23 +311,23 @@ end # ╔═╡ f742440b-258e-488a-9c8b-c9267cf1fb99 begin ncal = Int(conf_model.train_ratio * data_specs.N) - if model_evaluation.measurement[1] < coverage - Markdown.parse( - """ - > ❌❌❌ Oh no! You got an empirical coverage rate that is slightly lower than desired 🥲 ... what's happened? - - The coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). - """ - ) - else - Markdown.parse( - """ - > ✅ ✅ ✅ Great! You got an empirical coverage rate that is slightly higher than desired 😁 ... but why isn't it exactly the same? - - In most cases it will be slightly higher than desired, since ``(1-\\alpha)`` is a lower bound. But note that it can also be slightly lower than desired. That is because the coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). - """ - ) - end + if model_evaluation.measurement[1] < coverage + Markdown.parse( + """ + > ❌❌❌ Oh no! You got an empirical coverage rate that is slightly lower than desired 🥲 ... what's happened? + + The coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). + """, + ) + else + Markdown.parse( + """ + > ✅ ✅ ✅ Great! You got an empirical coverage rate that is slightly higher than desired 😁 ... but why isn't it exactly the same? + + In most cases it will be slightly higher than desired, since ``(1-\\alpha)`` is a lower bound. But note that it can also be slightly lower than desired. That is because the coverage property is "marginal" in the sense that the probability is averaged over the randomness in the data. For most purposes a large enough calibration set size (``n>1000``) mitigates that randomness enough. Depending on your choices above, the calibration set may be quite small (currently $ncal), which can lead to **coverage slack** (see Section 3 in the [tutorial](https://arxiv.org/pdf/2107.07511.pdf)). + """, + ) + end end # ╔═╡ f7b2296f-919f-4870-aac1-8e36dd694422 @@ -353,36 +359,40 @@ Quite cool, right? Using a single API call we are able to generate rigorous pred # ╔═╡ 824bd383-2fcb-4888-8ad1-260c85333edf @bind xmax_ood Slider( data_specs.xmax:(data_specs.xmax+5), - default=(data_specs.xmax), - show_value=true, + default = (data_specs.xmax), + show_value = true, ) # ╔═╡ 072cc72d-20a2-4ee9-954c-7ea70dfb8eea begin - Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun=f) - plot(_mach.model, _mach.fitresult, Xood, yood, zoom=0, observed_lab="Test points") - xood_range = range(-xmax_ood, xmax_ood, length=50) + Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun = f) + plot(_mach.model, _mach.fitresult, Xood, yood, zoom = 0, observed_lab = "Test points") + xood_range = range(-xmax_ood, xmax_ood, length = 50) plot!( xood_range, @.(f(xood_range)), - lw=2, - ls=:dash, - colour=:black, - label="Ground truth", + lw = 2, + ls = :dash, + colour = :black, + label = "Ground truth", ) end # ╔═╡ 4f41ec7c-aedd-475f-942d-33e2d1174902 if xmax_ood > data_specs.xmax - Markdown.parse(""" - > Whooooops 🤕 ... looks like we're in trouble! What happened here? - - By expaning the domain of out inputs, we have violated the exchangeability assumption. When that assumption is violated, the marginal coverage property does not hold. But do not despair! There are ways to deal with this. - """) + Markdown.parse( + """ +> Whooooops 🤕 ... looks like we're in trouble! What happened here? + +By expaning the domain of out inputs, we have violated the exchangeability assumption. When that assumption is violated, the marginal coverage property does not hold. But do not despair! There are ways to deal with this. +""", + ) else - Markdown.parse(""" - > Still looking OK 🤨 ... Try moving the slider above the chart to the right to see what will happen. - """) + Markdown.parse( + """ +> Still looking OK 🤨 ... Try moving the slider above the chart to the right to see what will happen. +""", + ) end # ╔═╡ c7fa1889-b0be-4d96-b845-e79fa7932b0c diff --git a/docs/pluto/understanding_coverage.jl b/docs/pluto/understanding_coverage.jl index 321f393..498f117 100644 --- a/docs/pluto/understanding_coverage.jl +++ b/docs/pluto/understanding_coverage.jl @@ -7,7 +7,14 @@ using InteractiveUtils # This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error). macro bind(def, element) quote - local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end + local iv = try + Base.loaded_modules[Base.PkgId( + Base.UUID("6e696c72-6542-2067-7265-42206c756150"), + "AbstractPlutoDingetjes", + )].Bonds.initial_value + catch + b -> missing + end local el = $(esc(element)) global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el) el @@ -16,7 +23,7 @@ end # ╔═╡ 435e65de-c2d5-436f-8c21-00fdee731959 begin - using ConformalPrediction + using ConformalPrediction using DecisionTree: DecisionTreeRegressor using Distributions using EvoTrees: EvoTreeRegressor @@ -38,25 +45,25 @@ md""" # ╔═╡ 59bdf205-b1d3-42ca-a256-f2878a87a691 # helper functions begin - function multi_slider(vals::Dict; title = "") - - return PlutoUI.combine() do Child - - inputs = [ - md""" $(_name): $( - Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) - )""" - - for (_name, _vals) in vals - ] - - md""" - #### *$title* - $(inputs) - """ - end - - end + function multi_slider(vals::Dict; title = "") + + return PlutoUI.combine() do Child + + inputs = [ + md""" $(_name): $( + Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) + )""" + + for (_name, _vals) in vals + ] + + md""" + #### *$title* + $(inputs) + """ + end + + end end; # ╔═╡ 7bad3b98-8172-4a06-a838-114d3954c594 @@ -69,10 +76,11 @@ We will use a simple helper function that generates some regression data with a # ╔═╡ 3dfc7d81-0644-4892-ba0d-7f3baba37ece begin function get_data( - N=600, noise=0.5; - fun::Function=fun(X) = X * sin(X), - d::Distribution=Distributions.Normal(0, 1) - ) + N = 600, + noise = 0.5; + fun::Function = fun(X) = X * sin(X), + d::Distribution = Distributions.Normal(0, 1), + ) # Inputs: X = rand(d, N) X = MLJBase.table(reshape(X, :, 1)) @@ -100,13 +108,13 @@ Below you can select the type of distribution that generates our input $X$: # ╔═╡ 39c53541-356a-4051-891e-4342887fd423 begin - dist_dict = Dict( - "Cauchy" => Cauchy, - "Cosine" => Cosine, - "Laplace" => Laplace, - "Normal" => Normal, - ) - @bind dist Select(collect(keys(dist_dict)), default="Normal") + dist_dict = Dict( + "Cauchy" => Cauchy, + "Cosine" => Cosine, + "Laplace" => Laplace, + "Normal" => Normal, + ) + @bind dist Select(collect(keys(dist_dict)), default = "Normal") end # ╔═╡ 6afc2fe2-e4ac-43d0-a4e5-ea4cd7ce82fc @@ -119,29 +127,29 @@ The chart below shows the resulting data (dots) and ground-truth according to ho # ╔═╡ 22ca7c3a-f06c-4a08-8805-27e0071cccb8 begin data_dict = Dict( - "N" => (100:100:2000, 1000), + "N" => (100:100:2000, 1000), "noise" => (0.1:0.1:1.0, 0.5), "location" => (-10000:1000:10000, 0), - "scale" => ([1,2,5,10,50,100,1000], 1), + "scale" => ([1, 2, 5, 10, 50, 100, 1000], 1), ) - @bind data_specs multi_slider(data_dict, title="Parameters") + @bind data_specs multi_slider(data_dict, title = "Parameters") end # ╔═╡ f5bbf50c-21d6-4aad-8d8f-781246382131 begin - d = dist_dict[dist](data_specs.location, data_specs.scale) - X, y = get_data(data_specs.N, data_specs.noise; fun=f, d=d) - train, test = partition(eachindex(y), 0.4, 0.4, shuffle=true) - scatter(X.x1, y, label="Observed data") - xrange = range(minimum(X.x1)[1], maximum(X.x1)[1], length=50) + d = dist_dict[dist](data_specs.location, data_specs.scale) + X, y = get_data(data_specs.N, data_specs.noise; fun = f, d = d) + train, test = partition(eachindex(y), 0.4, 0.4, shuffle = true) + scatter(X.x1, y, label = "Observed data") + xrange = range(minimum(X.x1)[1], maximum(X.x1)[1], length = 50) plot!( xrange, @.(f(xrange)), - lw=4, - label="Ground truth", - ls=:dash, - colour=:black, - size=(800,200) + lw = 4, + label = "Ground truth", + ls = :dash, + colour = :black, + size = (800, 200), ) end @@ -160,24 +168,29 @@ begin end; # ╔═╡ e6feb79d-f524-4d06-b18b-51ee6854c30b -@bind coverage Slider(0.1:0.01:1.0, default=0.8, show_value=true) +@bind coverage Slider(0.1:0.01:1.0, default = 0.8, show_value = true) # ╔═╡ bb75d0a7-c0b4-484c-b3da-224e368b743a begin - Xtest = MLJBase.matrix(selectrows(X, test)) + Xtest = MLJBase.matrix(selectrows(X, test)) ytest = y[test] - conf_model = conformal_model(model, coverage=coverage) + conf_model = conformal_model(model, coverage = coverage) mach = machine(conf_model, X, y) - MLJBase.fit!(mach, rows=train, verbosity=0) + MLJBase.fit!(mach, rows = train, verbosity = 0) s = conf_model.scores - α = 1 - conf_model.coverage - scatter( - s, zeros(length(s)); - ylim=(-0.5,0.5), yaxis=nothing, label="Scores", ms=10, alpha=0.2, - size=(800,150) - ) - q̂ = quantile(s, 1-α) - vline!([q̂], label="q̂") + α = 1 - conf_model.coverage + scatter( + s, + zeros(length(s)); + ylim = (-0.5, 0.5), + yaxis = nothing, + label = "Scores", + ms = 10, + alpha = 0.2, + size = (800, 150), + ) + q̂ = quantile(s, 1 - α) + vline!([q̂], label = "q̂") end # ╔═╡ 00000000-0000-0000-0000-000000000001 diff --git a/docs/setup_docs.jl b/docs/setup_docs.jl index 6a1563b..f12a6e6 100644 --- a/docs/setup_docs.jl +++ b/docs/setup_docs.jl @@ -19,7 +19,7 @@ setup_docs = quote using Transformers using Transformers.TextEncoders using Transformers.HuggingFace - + # Explicit imports: import MLJModelInterface as MMI import UnicodePlots diff --git a/src/ConformalPrediction.jl b/src/ConformalPrediction.jl index 509a9c7..8032cda 100644 --- a/src/ConformalPrediction.jl +++ b/src/ConformalPrediction.jl @@ -13,6 +13,6 @@ include("evaluation/evaluation.jl") export emp_coverage, size_stratified_coverage, ssc # Artifacts: -include("artifacts/core.jl") +include("artifacts/core.jl") end diff --git a/src/artifacts/core.jl b/src/artifacts/core.jl index 5fed867..e2e4732 100644 --- a/src/artifacts/core.jl +++ b/src/artifacts/core.jl @@ -10,4 +10,4 @@ function artifact_folder() @error "No artifacts found for Julia version $jversion." end return path -end \ No newline at end of file +end diff --git a/src/conformal_models/conformal_models.jl b/src/conformal_models/conformal_models.jl index 4289769..cd14825 100644 --- a/src/conformal_models/conformal_models.jl +++ b/src/conformal_models/conformal_models.jl @@ -64,11 +64,8 @@ include("transductive_classification.jl") include("training/training.jl") # Type unions: -const InductiveModel = Union{ - SimpleInductiveRegressor, - SimpleInductiveClassifier, - AdaptiveInductiveClassifier, -} +const InductiveModel = + Union{SimpleInductiveRegressor,SimpleInductiveClassifier,AdaptiveInductiveClassifier} const TransductiveModel = Union{ NaiveRegressor, diff --git a/src/conformal_models/inductive_classification.jl b/src/conformal_models/inductive_classification.jl index 9334519..031f03b 100644 --- a/src/conformal_models/inductive_classification.jl +++ b/src/conformal_models/inductive_classification.jl @@ -3,7 +3,12 @@ Generic score method for the [`ConformalProbabilisticSet`](@ref). It computes nonconformity scores using the heuristic function `h` and the softmax probabilities of the true class. Method is dispatched for different Conformal Probabilistic Sets and atomic models. """ -function score(conf_model::ConformalProbabilisticSet, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) +function score( + conf_model::ConformalProbabilisticSet, + fitresult, + X, + y::Union{Nothing,AbstractArray} = nothing, +) score(conf_model, typeof(conf_model.model), fitresult, X, y) end @@ -35,9 +40,9 @@ end function SimpleInductiveClassifier( model::Supervised; - coverage::AbstractFloat=0.95, - heuristic::Function=f(p̂) = 1.0 - p̂, - train_ratio::AbstractFloat=0.5, + coverage::AbstractFloat = 0.95, + heuristic::Function = f(p̂) = 1.0 - p̂, + train_ratio::AbstractFloat = 0.5, ) return SimpleInductiveClassifier(model, coverage, nothing, heuristic, train_ratio) end @@ -47,7 +52,13 @@ end Score method for the [`SimpleInductiveClassifier`](@ref) dispatched for any `<:Supervised` model. """ -function score(conf_model::SimpleInductiveClassifier, ::Type{<:Supervised}, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) +function score( + conf_model::SimpleInductiveClassifier, + ::Type{<:Supervised}, + fitresult, + X, + y::Union{Nothing,AbstractArray} = nothing, +) p̂ = reformat_mlj_prediction(MMI.predict(conf_model.model, fitresult, X)) L = p̂.decoder.classes probas = pdf(p̂, L) @@ -81,10 +92,7 @@ function MMI.fit(conf_model::SimpleInductiveClassifier, verbosity, X, y) # Nonconformity Scores: cal_scores, scores = score(conf_model, fitresult, Xcal, ycal) - conf_model.scores = Dict( - :calibration => cal_scores, - :all => scores, - ) + conf_model.scores = Dict(:calibration => cal_scores, :all => scores) return (fitresult, cache, report) end @@ -132,9 +140,9 @@ end function AdaptiveInductiveClassifier( model::Supervised; - coverage::AbstractFloat=0.95, - heuristic::Function=f(y, ŷ) = 1.0 - ŷ, - train_ratio::AbstractFloat=0.5, + coverage::AbstractFloat = 0.95, + heuristic::Function = f(y, ŷ) = 1.0 - ŷ, + train_ratio::AbstractFloat = 0.5, ) return AdaptiveInductiveClassifier(model, coverage, nothing, heuristic, train_ratio) end @@ -158,10 +166,7 @@ function MMI.fit(conf_model::AdaptiveInductiveClassifier, verbosity, X, y) # Nonconformity Scores: cal_scores, scores = score(conf_model, fitresult, Xcal, ycal) - conf_model.scores = Dict( - :calibration => cal_scores, - :all => scores, - ) + conf_model.scores = Dict(:calibration => cal_scores, :all => scores) return (fitresult, cache, report) end @@ -171,7 +176,13 @@ end Score method for the [`AdaptiveInductiveClassifier`](@ref) dispatched for any `<:Supervised` model. """ -function score(conf_model::AdaptiveInductiveClassifier, ::Type{<:Supervised}, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) +function score( + conf_model::AdaptiveInductiveClassifier, + ::Type{<:Supervised}, + fitresult, + X, + y::Union{Nothing,AbstractArray} = nothing, +) p̂ = reformat_mlj_prediction(MMI.predict(conf_model.model, fitresult, X)) L = p̂.decoder.classes probas = pdf(p̂, L) # compute probabilities for all classes diff --git a/src/conformal_models/plotting.jl b/src/conformal_models/plotting.jl index d8093b0..f7c4f12 100644 --- a/src/conformal_models/plotting.jl +++ b/src/conformal_models/plotting.jl @@ -97,7 +97,7 @@ function Plots.contourf( @info "No target label supplied, using first." end target = isnothing(target) ? levels(y)[1] : target - if plot_set_size + if plot_set_size _default_title = "Set size" elseif plot_set_loss _default_title = "Smooth set loss" @@ -107,7 +107,7 @@ function Plots.contourf( _default_title = "p̂(y=$(target))" end else - if plot_set_size + if plot_set_size _default_title = "Set size" elseif plot_set_loss _default_title = "Smooth set loss" @@ -126,10 +126,23 @@ function Plots.contourf( if plot_set_size z = ismissing(p̂) ? 0 : sum(pdf.(p̂, p̂.decoder.classes) .> 0) elseif plot_classification_loss - _target = categorical([target], levels=levels(y)) - z = ConformalPrediction.classification_loss(conf_model, fitresult, [x1 x2], _target; temp=temp, loss_matrix=loss_matrix) + _target = categorical([target], levels = levels(y)) + z = ConformalPrediction.classification_loss( + conf_model, + fitresult, + [x1 x2], + _target; + temp = temp, + loss_matrix = loss_matrix, + ) elseif plot_set_loss - z = ConformalPrediction.smooth_size_loss(conf_model, fitresult, [x1 x2]; κ=κ, temp=temp) + z = ConformalPrediction.smooth_size_loss( + conf_model, + fitresult, + [x1 x2]; + κ = κ, + temp = temp, + ) else z = ismissing(p̂) ? [missing for i = 1:length(levels(y))] : pdf.(p̂, levels(y)) z = replace(z, 0 => missing) @@ -165,7 +178,7 @@ function Plots.contourf( ylims = ylims, clim = clim, c = cgrad(:blues), - linewidth=0, + linewidth = 0, kwargs..., ) end @@ -331,4 +344,4 @@ function Plots.bar( x = sort(levels(idx), lt = natural) y = [sum(idx .== _x) for _x in x] Plots.bar(x, y; label = label, xtickfontsize = xtickfontsize, kwrgs...) -end \ No newline at end of file +end diff --git a/src/conformal_models/training/inductive_classification.jl b/src/conformal_models/training/inductive_classification.jl index c25101c..589d373 100644 --- a/src/conformal_models/training/inductive_classification.jl +++ b/src/conformal_models/training/inductive_classification.jl @@ -7,7 +7,13 @@ using MLUtils Overloads the `score` function for the `MLJFluxModel` type. """ -function score(conf_model::SimpleInductiveClassifier, ::Type{<:MLJFluxModel}, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) +function score( + conf_model::SimpleInductiveClassifier, + ::Type{<:MLJFluxModel}, + fitresult, + X, + y::Union{Nothing,AbstractArray} = nothing, +) X = reformat(X) X = typeof(X) <: AbstractArray ? X : permutedims(matrix(X)) probas = permutedims(fitresult[1](X)) @@ -25,14 +31,21 @@ end Overloads the `score` function for ensembles of `MLJFluxModel` types. """ -function score(conf_model::SimpleInductiveClassifier, ::Type{<:EitherEnsembleModel{<:MLJFluxModel}}, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) +function score( + conf_model::SimpleInductiveClassifier, + ::Type{<:EitherEnsembleModel{<:MLJFluxModel}}, + fitresult, + X, + y::Union{Nothing,AbstractArray} = nothing, +) X = reformat(X) X = typeof(X) <: AbstractArray ? X : permutedims(matrix(X)) _chains = map(res -> res[1], fitresult.ensemble) - probas = MLUtils.stack(map(chain -> chain(X), _chains)) |> - p -> mean(p, dims=ndims(p)) |> - p -> MLUtils.unstack(p, dims=ndims(p))[1] |> - p -> permutedims(p) + probas = + MLUtils.stack(map(chain -> chain(X), _chains)) |> + p -> + mean(p, dims = ndims(p)) |> + p -> MLUtils.unstack(p, dims = ndims(p))[1] |> p -> permutedims(p) scores = @.(conf_model.heuristic(probas)) if isnothing(y) return scores @@ -47,7 +60,13 @@ end Overloads the `score` function for the `MLJFluxModel` type. """ -function score(conf_model::AdaptiveInductiveClassifier, ::Type{<:MLJFluxModel}, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) +function score( + conf_model::AdaptiveInductiveClassifier, + ::Type{<:MLJFluxModel}, + fitresult, + X, + y::Union{Nothing,AbstractArray} = nothing, +) L = levels(fitresult[2]) X = reformat(X) X = typeof(X) <: AbstractArray ? X : permutedims(matrix(X)) @@ -72,15 +91,22 @@ end Overloads the `score` function for ensembles of `MLJFluxModel` types. """ -function score(conf_model::AdaptiveInductiveClassifier, ::Type{<:EitherEnsembleModel{<:MLJFluxModel}}, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) +function score( + conf_model::AdaptiveInductiveClassifier, + ::Type{<:EitherEnsembleModel{<:MLJFluxModel}}, + fitresult, + X, + y::Union{Nothing,AbstractArray} = nothing, +) L = levels(fitresult.ensemble[1][2]) X = reformat(X) X = typeof(X) <: AbstractArray ? X : permutedims(matrix(X)) _chains = map(res -> res[1], fitresult.ensemble) - probas = MLUtils.stack(map(chain -> chain(X), _chains)) |> - p -> mean(p, dims=ndims(p)) |> - p -> MLUtils.unstack(p, dims=ndims(p))[1] |> - p -> permutedims(p) + probas = + MLUtils.stack(map(chain -> chain(X), _chains)) |> + p -> + mean(p, dims = ndims(p)) |> + p -> MLUtils.unstack(p, dims = ndims(p))[1] |> p -> permutedims(p) scores = map(Base.Iterators.product(eachrow(probas), L)) do Z probasᵢ, yₖ = Z ranks = sortperm(.-probasᵢ) # rank in descending order @@ -94,4 +120,4 @@ function score(conf_model::AdaptiveInductiveClassifier, ::Type{<:EitherEnsembleM cal_scores = getindex.(Ref(scores), 1:size(scores, 1), levelcode.(y)) return cal_scores, scores end -end \ No newline at end of file +end diff --git a/src/conformal_models/training/losses.jl b/src/conformal_models/training/losses.jl index 530b895..2b69045 100644 --- a/src/conformal_models/training/losses.jl +++ b/src/conformal_models/training/losses.jl @@ -7,10 +7,13 @@ using MLJBase Computes soft assignment scores for each label and sample. That is, the probability of label `k` being included in the confidence set. This implementation follows Stutz et al. (2022): https://openreview.net/pdf?id=t8O-4LKFVx. Contrary to the paper, we use non-conformity scores instead of conformity scores, hence the sign swap. """ -function soft_assignment(conf_model::ConformalProbabilisticSet; temp::Union{Nothing, Real}=nothing) +function soft_assignment( + conf_model::ConformalProbabilisticSet; + temp::Union{Nothing,Real} = nothing, +) temp = isnothing(temp) ? 0.5 : temp v = sort(conf_model.scores[:calibration]) - q̂ = StatsBase.quantile(v, conf_model.coverage, sorted=true) + q̂ = StatsBase.quantile(v, conf_model.coverage, sorted = true) scores = conf_model.scores[:all] return @.(σ((q̂ - scores) / temp)) end @@ -20,10 +23,15 @@ end This function can be used to compute soft assigment probabilities for new data `X` as in [`soft_assignment(conf_model::ConformalProbabilisticSet; temp::Real=0.5)`](@ref). When a fitted model $\mu$ (`fitresult`) and new samples `X` are supplied, non-conformity scores are first computed for the new data points. Then the existing threshold/quantile `q̂` is used to compute the final soft assignments. """ -function soft_assignment(conf_model::ConformalProbabilisticSet, fitresult, X; temp::Union{Nothing, Real}=nothing) +function soft_assignment( + conf_model::ConformalProbabilisticSet, + fitresult, + X; + temp::Union{Nothing,Real} = nothing, +) temp = isnothing(temp) ? 0.5 : temp v = sort(conf_model.scores[:calibration]) - q̂ = StatsBase.quantile(v, conf_model.coverage, sorted=true) + q̂ = StatsBase.quantile(v, conf_model.coverage, sorted = true) scores = score(conf_model, fitresult, X) return @.(σ((q̂ - scores) / temp)) end @@ -43,16 +51,20 @@ Computes the smooth (differentiable) size loss following Stutz et al. (2022): ht where $\tau$ is just the quantile `q̂` and $\kappa$ is the target set size (defaults to $1$). For empty sets, the loss is computed as $K - \kappa$, that is the maximum set size minus the target set size. """ function smooth_size_loss( - conf_model::ConformalProbabilisticSet, fitresult, X; - temp::Union{Nothing, Real}=nothing, κ::Real=1.0 + conf_model::ConformalProbabilisticSet, + fitresult, + X; + temp::Union{Nothing,Real} = nothing, + κ::Real = 1.0, ) temp = isnothing(temp) ? 0.5 : temp - C = soft_assignment(conf_model, fitresult, X; temp=temp) - is_empty_set = all(x -> x .== 0, soft_assignment(conf_model, fitresult, X; temp=0.0), dims=2) + C = soft_assignment(conf_model, fitresult, X; temp = temp) + is_empty_set = + all(x -> x .== 0, soft_assignment(conf_model, fitresult, X; temp = 0.0), dims = 2) Ω = [] for i in eachindex(is_empty_set) - c = C[i,:] - if is_empty_set[i] + c = C[i, :] + if is_empty_set[i] x = sum(1 .- c) else x = sum(c) @@ -88,23 +100,26 @@ Computes the calibration loss following Stutz et al. (2022): https://openreview. where $\tau$ is just the quantile `q̂` and $\kappa$ is the target set size (defaults to $1$). """ function classification_loss( - conf_model::ConformalProbabilisticSet, fitresult, X, y; - loss_matrix::Union{AbstractMatrix,UniformScaling}=UniformScaling(1.0), - temp::Union{Nothing, Real}=nothing + conf_model::ConformalProbabilisticSet, + fitresult, + X, + y; + loss_matrix::Union{AbstractMatrix,UniformScaling} = UniformScaling(1.0), + temp::Union{Nothing,Real} = nothing, ) # Setup: temp = isnothing(temp) ? 0.5 : temp if typeof(y) <: CategoricalArray L = levels(y) - yenc = permutedims(Flux.onehotbatch(levelcode.(y), L)) - else + yenc = permutedims(Flux.onehotbatch(levelcode.(y), L)) + else yenc = y end K = size(yenc, 2) if typeof(loss_matrix) <: UniformScaling loss_matrix = Matrix(loss_matrix(K)) end - C = soft_assignment(conf_model, fitresult, X; temp=temp) + C = soft_assignment(conf_model, fitresult, X; temp = temp) # Loss: ℒ = map(eachrow(C), eachrow(yenc)) do c, _yenc @@ -117,4 +132,4 @@ function classification_loss( ℒ = reduce(vcat, ℒ) ℒ = permutedims(permutedims(ℒ)) return ℒ -end \ No newline at end of file +end diff --git a/src/conformal_models/training/training.jl b/src/conformal_models/training/training.jl index 808639e..8fd1da3 100644 --- a/src/conformal_models/training/training.jl +++ b/src/conformal_models/training/training.jl @@ -1,2 +1,2 @@ include("losses.jl") -include("inductive_classification.jl") \ No newline at end of file +include("inductive_classification.jl") diff --git a/src/conformal_models/transductive_classification.jl b/src/conformal_models/transductive_classification.jl index ea58387..9227dd5 100644 --- a/src/conformal_models/transductive_classification.jl +++ b/src/conformal_models/transductive_classification.jl @@ -77,4 +77,4 @@ function MMI.predict(conf_model::NaiveClassifier, fitresult, Xnew) return pp end return p̂ -end \ No newline at end of file +end diff --git a/src/conformal_models/transductive_regression.jl b/src/conformal_models/transductive_regression.jl index 89e1849..dfaf361 100644 --- a/src/conformal_models/transductive_regression.jl +++ b/src/conformal_models/transductive_regression.jl @@ -554,7 +554,7 @@ function _aggregate(y, aggregate::Union{Symbol,String}) valid_methods = Dict( :mean => x -> StatsBase.mean(x), :median => x -> StatsBase.median(x), - :trimmedmean => x -> StatsBase.mean(trim(x, prop=0.1)), + :trimmedmean => x -> StatsBase.mean(trim(x, prop = 0.1)), ) @assert aggregate ∈ keys(valid_methods) "`aggregate`=$aggregate is not a valid aggregation method. Should be one of: $valid_methods" # Aggregate: @@ -569,7 +569,7 @@ end # Jackknife_plus_after_bootstrapping "Constructor for `JackknifePlusAbPlusRegressor`." -mutable struct JackknifePlusAbRegressor{Model <: Supervised} <: ConformalInterval +mutable struct JackknifePlusAbRegressor{Model<:Supervised} <: ConformalInterval model::Model coverage::AbstractFloat scores::Union{Nothing,AbstractArray} @@ -577,19 +577,28 @@ mutable struct JackknifePlusAbRegressor{Model <: Supervised} <: ConformalInterva nsampling::Int sample_size::AbstractFloat replacement::Bool - aggregate::Union{Symbol, String} + aggregate::Union{Symbol,String} end function JackknifePlusAbRegressor( - model::Supervised; - coverage::AbstractFloat=0.95, - heuristic::Function=f(y,ŷ)=abs(y-ŷ), - nsampling::Int=30, - sample_size::AbstractFloat=0.5, - replacement::Bool=true, - aggregate::Union{Symbol, String}="mean" + model::Supervised; + coverage::AbstractFloat = 0.95, + heuristic::Function = f(y, ŷ) = abs(y - ŷ), + nsampling::Int = 30, + sample_size::AbstractFloat = 0.5, + replacement::Bool = true, + aggregate::Union{Symbol,String} = "mean", ) - return JackknifePlusAbRegressor(model, coverage, nothing, heuristic, nsampling, sample_size, replacement, aggregate) + return JackknifePlusAbRegressor( + model, + coverage, + nothing, + heuristic, + nsampling, + sample_size, + replacement, + aggregate, + ) end @doc raw""" @@ -604,33 +613,38 @@ $ S_i^{\text{J+ab}} = s(X_i, Y_i) = h(agg(\hat\mu_{B_{K(-i)}}(X_i)), Y_i), \ i \ where ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` denotes the aggregate predictions, typically mean or median, for each ``X_i`` (with ``K_{-i}`` the bootstraps not containing ``X_i``). In other words, B models are trained on boostrapped sampling, the fitted models are then used to create aggregated prediction of out-of-sample ``X_i``. The corresponding nonconformity score is then computed by applying a heuristic uncertainty measure ``h(\cdot)`` to the fitted value ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` and the true value ``Y_i``. """ function MMI.fit(conf_model::JackknifePlusAbRegressor, verbosity, X, y) - - samples, fitresult, cache, report, scores = ([],[],[],[],[]) + + samples, fitresult, cache, report, scores = ([], [], [], [], []) replacement = conf_model.replacement nsampling = conf_model.nsampling sample_size = conf_model.sample_size aggregate = conf_model.aggregate - T = size(y,1) + T = size(y, 1) # bootstrap size - m = floor(Int, T* sample_size) - for _ in 1:nsampling - samplesᵢ = sample(1:T, m, replace=replacement) - yᵢ = y[samplesᵢ] + m = floor(Int, T * sample_size) + for _ = 1:nsampling + samplesᵢ = sample(1:T, m, replace = replacement) + yᵢ = y[samplesᵢ] Xᵢ = selectrows(X, samplesᵢ) - μ̂ᵢ, cacheᵢ, reportᵢ = MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)...) + μ̂ᵢ, cacheᵢ, reportᵢ = + MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)...) push!(samples, samplesᵢ) push!(fitresult, μ̂ᵢ) push!(cache, cacheᵢ) push!(report, reportᵢ) end - for t in 1:T + for t = 1:T index_samples = indexin([v for v in samples if !(t in v)], samples) selected_models = fitresult[index_samples] Xₜ = selectrows(X, t) yₜ = y[t] - ŷ = [reformat_mlj_prediction(MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...)) for μ̂₋ₜ in selected_models] + ŷ = [ + reformat_mlj_prediction( + MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...), + ) for μ̂₋ₜ in selected_models + ] ŷₜ = _aggregate(ŷ, aggregate) - push!(scores,@.(conf_model.heuristic(yₜ, ŷₜ))...) + push!(scores, @.(conf_model.heuristic(yₜ, ŷₜ))...) end scores = filter(!isnan, scores) conf_model.scores = scores @@ -651,7 +665,11 @@ where ``\hat\mu_{agg(-i)}`` denotes the aggregated models ``\hat\mu_{1}, ...., \ """ function MMI.predict(conf_model::JackknifePlusAbRegressor, fitresult, Xnew) # Get all bootstrapped predictions for each Xnew: - ŷ = [reformat_mlj_prediction(MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...)) for μ̂₋ᵢ in fitresult] + ŷ = [ + reformat_mlj_prediction( + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + ) for μ̂₋ᵢ in fitresult + ] # Applying aggregation function on bootstrapped predictions across columns for each Xnew across rows: aggregate = conf_model.aggregate ŷ = _aggregate(ŷ, aggregate) @@ -664,7 +682,7 @@ end # Jackknife_plus_after_bootstrapping_minmax "Constructor for `JackknifePlusAbPlusMinMaxRegressor`." -mutable struct JackknifePlusAbMinMaxRegressor{Model <: Supervised} <: ConformalInterval +mutable struct JackknifePlusAbMinMaxRegressor{Model<:Supervised} <: ConformalInterval model::Model coverage::AbstractFloat scores::Union{Nothing,AbstractArray} @@ -672,19 +690,28 @@ mutable struct JackknifePlusAbMinMaxRegressor{Model <: Supervised} <: ConformalI nsampling::Int sample_size::AbstractFloat replacement::Bool - aggregate::Union{Symbol, String} + aggregate::Union{Symbol,String} end function JackknifePlusAbMinMaxRegressor( - model::Supervised; - coverage::AbstractFloat=0.95, - heuristic::Function=f(y,ŷ)=abs(y-ŷ), - nsampling::Int=30, - sample_size::AbstractFloat=0.5, - replacement::Bool=true, - aggregate::Union{Symbol, String}="mean" + model::Supervised; + coverage::AbstractFloat = 0.95, + heuristic::Function = f(y, ŷ) = abs(y - ŷ), + nsampling::Int = 30, + sample_size::AbstractFloat = 0.5, + replacement::Bool = true, + aggregate::Union{Symbol,String} = "mean", ) - return JackknifePlusAbMinMaxRegressor(model, coverage, nothing, heuristic, nsampling, sample_size, replacement, aggregate) + return JackknifePlusAbMinMaxRegressor( + model, + coverage, + nothing, + heuristic, + nsampling, + sample_size, + replacement, + aggregate, + ) end @doc raw""" @@ -699,33 +726,38 @@ S_i^{\text{J+MinMax}} = s(X_i, Y_i) = h(agg(\hat\mu_{B_{K(-i)}}(X_i)), Y_i), \ i where ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` denotes the aggregate predictions, typically mean or median, for each ``X_i`` (with ``K_{-i}`` the bootstraps not containing ``X_i``). In other words, B models are trained on boostrapped sampling, the fitted models are then used to create aggregated prediction of out-of-sample ``X_i``. The corresponding nonconformity score is then computed by applying a heuristic uncertainty measure ``h(\cdot)`` to the fitted value ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` and the true value ``Y_i``. """ function MMI.fit(conf_model::JackknifePlusAbMinMaxRegressor, verbosity, X, y) - - samples, fitresult, cache, report, scores = ([],[],[],[],[]) + + samples, fitresult, cache, report, scores = ([], [], [], [], []) replacement = conf_model.replacement nsampling = conf_model.nsampling sample_size = conf_model.sample_size aggregate = conf_model.aggregate - T = size(y,1) + T = size(y, 1) # bootstrap size - m = floor(Int, T*sample_size) - for _ in 1:nsampling - samplesᵢ = sample(1:T, m, replace=replacement) - yᵢ = y[samplesᵢ] + m = floor(Int, T * sample_size) + for _ = 1:nsampling + samplesᵢ = sample(1:T, m, replace = replacement) + yᵢ = y[samplesᵢ] Xᵢ = selectrows(X, samplesᵢ) - μ̂ᵢ, cacheᵢ, reportᵢ = MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)...) + μ̂ᵢ, cacheᵢ, reportᵢ = + MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)...) push!(samples, samplesᵢ) push!(fitresult, μ̂ᵢ) push!(cache, cacheᵢ) push!(report, reportᵢ) end - for t in 1:T + for t = 1:T index_samples = indexin([v for v in samples if !(t in v)], samples) selected_models = fitresult[index_samples] Xₜ = selectrows(X, t) yₜ = y[t] - ŷ = [reformat_mlj_prediction(MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...)) for μ̂₋ₜ in selected_models] + ŷ = [ + reformat_mlj_prediction( + MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...), + ) for μ̂₋ₜ in selected_models + ] ŷₜ = _aggregate(ŷ, aggregate) - push!(scores,@.(conf_model.heuristic(yₜ, ŷₜ))...) + push!(scores, @.(conf_model.heuristic(yₜ, ŷₜ))...) end scores = filter(!isnan, scores) conf_model.scores = scores @@ -746,7 +778,11 @@ where ``\hat\mu_{-i}`` denotes the model fitted on training data with ``i``th po """ function MMI.predict(conf_model::JackknifePlusAbMinMaxRegressor, fitresult, Xnew) # Get all bootstrapped predictions for each Xnew: - ŷ = [reformat_mlj_prediction(MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...)) for μ̂₋ᵢ in fitresult] + ŷ = [ + reformat_mlj_prediction( + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + ) for μ̂₋ᵢ in fitresult + ] ŷ = reduce(hcat, ŷ) v = conf_model.scores q̂ = StatsBase.quantile(v, conf_model.coverage) From 34a64edd45021e087f1793503906d162d3957645 Mon Sep 17 00:00:00 2001 From: pat-alt Date: Thu, 6 Jul 2023 16:00:58 +0200 Subject: [PATCH 3/3] formatter --- .JuliaFormatter.jl | 8 +- .JuliaFormatter.toml | 2 + dev/logo/logo.jl | 85 +++++---- docs/make.jl | 27 ++- docs/pluto/intro.jl | 86 +++------ docs/pluto/intro_dev.jl | 89 +++------- docs/pluto/understanding_coverage.jl | 62 +++---- docs/setup_docs.jl | 3 +- src/conformal_models/conformal_models.jl | 18 +- .../inductive_classification.jl | 24 +-- src/conformal_models/inductive_regression.jl | 8 +- src/conformal_models/plotting.jl | 119 +++++-------- .../training/inductive_classification.jl | 16 +- src/conformal_models/training/losses.jl | 29 ++- .../transductive_classification.jl | 7 +- .../transductive_regression.jl | 167 ++++++++---------- src/conformal_models/utils.jl | 18 +- src/evaluation/measures.jl | 3 +- test/classification.jl | 38 ++-- test/model_traits.jl | 6 +- test/regression.jl | 33 ++-- test/runtests.jl | 2 - 22 files changed, 350 insertions(+), 500 deletions(-) create mode 100644 .JuliaFormatter.toml diff --git a/.JuliaFormatter.jl b/.JuliaFormatter.jl index 323237b..88e7e4b 100644 --- a/.JuliaFormatter.jl +++ b/.JuliaFormatter.jl @@ -1 +1,7 @@ -style = "blue" +using Pkg # Load package manager +Pkg.add("JuliaFormatter") # Install JuliaFormatter + +using JuliaFormatter # Load JuliaFormatter +format("."; verbose=true) # Format all files + +Pkg.rm("JuliaFormatter") # Remove JuliaFormatter diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml new file mode 100644 index 0000000..8f9f54f --- /dev/null +++ b/.JuliaFormatter.toml @@ -0,0 +1,2 @@ +style = "blue" +pipe_to_function_call = false \ No newline at end of file diff --git a/dev/logo/logo.jl b/dev/logo/logo.jl index 5a6a077..7217e87 100644 --- a/dev/logo/logo.jl +++ b/dev/logo/logo.jl @@ -19,7 +19,7 @@ const julia_colors = Dict( :purple => Luxor.julia_purple, ) -function get_data(N = 500; xmax = 2.0, noise = 0.5, fun::Function = f) +function get_data(N=500; xmax=2.0, noise=0.5, fun::Function=f) # Inputs: d = Distributions.Uniform(-xmax, xmax) x = rand(d, N) @@ -32,21 +32,21 @@ function get_data(N = 500; xmax = 2.0, noise = 0.5, fun::Function = f) end function logo_picture(; - ndots = 3, - frame_size = 500, - ms = frame_size // 10, - mcolor = (:red, :green, :purple), - margin = 0.1, - fun = f(x) = x * cos(x), - xmax = 2.5, - noise = 0.5, - ged_data = get_data, - ntrue = 50, - gt_color = julia_colors[:blue], - gt_stroke_size = 5, - interval_color = julia_colors[:blue], - interval_alpha = 0.2, - seed = 2022, + ndots=3, + frame_size=500, + ms=frame_size//10, + mcolor=(:red, :green, :purple), + margin=0.1, + fun=f(x) = x * cos(x), + xmax=2.5, + noise=0.5, + ged_data=get_data, + ntrue=50, + gt_color=julia_colors[:blue], + gt_stroke_size=5, + interval_color=julia_colors[:blue], + interval_alpha=0.2, + seed=2022, ) # Setup @@ -55,8 +55,8 @@ function logo_picture(; Random.seed!(seed) # Data - x, y = get_data(xmax = xmax, noise = noise, fun = fun) - train, test = partition(eachindex(y), 0.4, 0.4, shuffle = true) + x, y = get_data(; xmax=xmax, noise=noise, fun=fun) + train, test = partition(eachindex(y), 0.4, 0.4; shuffle=true) xtrue = range(-xmax, xmax, ntrue) ytrue = fun.(xtrue) @@ -65,15 +65,15 @@ function logo_picture(; degree_polynomial = 5 polynomial_features(x, degree::Int) = reduce(hcat, map(i -> x .^ i, 1:degree)) pipe = (x -> MLJBase.table(polynomial_features(x, degree_polynomial))) |> Model() - conf_model = conformal_model(pipe; coverage = 0.95) + conf_model = conformal_model(pipe; coverage=0.95) mach = machine(conf_model, x, y) - fit!(mach, rows = train) + fit!(mach; rows=train) yhat = predict(mach, x[test]) y_lb = [y[1] for y in yhat] y_ub = [y[2] for y in yhat] # Logo - idx = sample(test, ndots, replace = false) + idx = sample(test, ndots; replace=false) xplot, yplot = (x[idx], y[idx]) _scale = (frame_size / (2 * maximum(x))) * (1 - margin) @@ -81,15 +81,15 @@ function logo_picture(; setline(gt_stroke_size) sethue(gt_color) true_points = [Point((_scale .* (x, y))...) for (x, y) in zip(xtrue, ytrue)] - poly(true_points[1:(end-1)], action = :stroke) + poly(true_points[1:(end - 1)]; action=:stroke) # Data data_plot = zip(xplot, yplot) - for i = 1:length(data_plot) + for i in 1:length(data_plot) _x, _y = _scale .* collect(data_plot)[i] color_idx = i % n_mcolor == 0 ? n_mcolor : i % n_mcolor sethue(mcolor[color_idx]...) - circle(Point(_x, _y), ms, action = :fill) + circle(Point(_x, _y), ms; action=:fill) end # Prediction interval: @@ -102,27 +102,26 @@ function logo_picture(; Point((_scale .* (x, y))...) for (x, y) in zip(x[test][_order_ub], y_ub[_order_ub]) ] setcolor(sethue(interval_color)..., interval_alpha) - poly(vcat(lb, ub), action = :fill) - + return poly(vcat(lb, ub); action=:fill) end -function draw_small_logo(filename = "docs/src/assets/logo.svg"; width = 500) +function draw_small_logo(filename="docs/src/assets/logo.svg"; width=500) frame_size = width Drawing(frame_size, frame_size, filename) origin() - logo_picture(frame_size = frame_size) + logo_picture(; frame_size=frame_size) finish() - preview() + return preview() end function draw_wide_logo_new( - filename = "docs/src/assets/wide_logo.png"; - _pkg_name = "Conformal Prediction", - font_size = 150, - font_family = "Tamil MN", - font_fill = "transparent", - font_color = Luxor.julia_blue, - bg_color = "transparent", + filename="docs/src/assets/wide_logo.png"; + _pkg_name="Conformal Prediction", + font_size=150, + font_family="Tamil MN", + font_fill="transparent", + font_color=Luxor.julia_blue, + bg_color="transparent", picture_kwargs..., ) @@ -145,11 +144,11 @@ function draw_wide_logo_new( # Picture: @layer begin translate(cells[1]) - logo_picture( - frame_size = height, - margin = 0.1, - ms = ms, - gt_stroke_size = gt_stroke_size, + logo_picture(; + frame_size=height, + margin=0.1, + ms=ms, + gt_stroke_size=gt_stroke_size, picture_kwargs..., ) end @@ -165,7 +164,7 @@ function draw_wide_logo_new( translate(pos) setline(Int(round(gt_stroke_size / 5))) sethue(font_fill) - textoutlines(strs[n], O, :path, valign = :middle, halign = :center) + textoutlines(strs[n], O, :path; valign=:middle, halign=:center) sethue(font_color) strokepath() end @@ -173,7 +172,7 @@ function draw_wide_logo_new( end finish() - preview() + return preview() end draw_wide_logo_new() diff --git a/docs/make.jl b/docs/make.jl index edcf871..92bdb99 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -17,20 +17,20 @@ ex_meta = quote model = DecisionTreeRegressor() end -DocMeta.setdocmeta!(ConformalPrediction, :DocTestSetup, ex_meta; recursive = true) +DocMeta.setdocmeta!(ConformalPrediction, :DocTestSetup, ex_meta; recursive=true) makedocs(; - modules = [ConformalPrediction], - authors = "Patrick Altmeyer", - repo = "https://github.com/juliatrustworthyai/ConformalPrediction.jl/blob/{commit}{path}#{line}", - sitename = "ConformalPrediction.jl", - format = Documenter.HTML(; - prettyurls = get(ENV, "CI", "false") == "true", - canonical = "https://juliatrustworthyai.github.io/ConformalPrediction.jl", - edit_link = "main", - assets = String[], + modules=[ConformalPrediction], + authors="Patrick Altmeyer", + repo="https://github.com/juliatrustworthyai/ConformalPrediction.jl/blob/{commit}{path}#{line}", + sitename="ConformalPrediction.jl", + format=Documenter.HTML(; + prettyurls=get(ENV, "CI", "false") == "true", + canonical="https://juliatrustworthyai.github.io/ConformalPrediction.jl", + edit_link="main", + assets=String[], ), - pages = [ + pages=[ "🏠 Home" => "index.md", "🫣 Tutorials" => [ "Overview" => "tutorials/index.md", @@ -53,7 +53,4 @@ makedocs(; ], ) -deploydocs(; - repo = "github.com/JuliaTrustworthyAI/ConformalPrediction.jl", - devbranch = "main", -) +deploydocs(; repo="github.com/JuliaTrustworthyAI/ConformalPrediction.jl", devbranch="main") diff --git a/docs/pluto/intro.jl b/docs/pluto/intro.jl index 08ebd3e..318b09d 100644 --- a/docs/pluto/intro.jl +++ b/docs/pluto/intro.jl @@ -51,10 +51,8 @@ Let's start by loading the necessary packages: # ╔═╡ 55a7c16b-a526-41d9-9d73-a0591ad006ce # helper functions begin - function multi_slider(vals::Dict; title = "") - + function multi_slider(vals::Dict; title="") return PlutoUI.combine() do Child - inputs = [ md""" $(_name): $( Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) @@ -68,7 +66,6 @@ begin $(inputs) """ end - end end; @@ -96,7 +93,7 @@ Most machine learning workflows start with data. In this tutorial you have full # ╔═╡ 2f1c8da3-77dc-4bd7-8fa4-7669c2861aaa begin - function get_data(N = 600, xmax = 3.0, noise = 0.5; fun::Function = fun(X) = X * sin(X)) + function get_data(N=600, xmax=3.0, noise=0.5; fun::Function=fun(X) = X * sin(X)) # Inputs: d = Distributions.Uniform(-xmax, xmax) X = rand(d, N) @@ -126,26 +123,17 @@ The sliders below can be used to change the number of observations `N`, the maxi # ╔═╡ 931ce259-d5fb-4a56-beb8-61a69a2fc09e begin data_dict = Dict( - "N" => (100:100:2000, 1000), - "noise" => (0.1:0.1:1.0, 0.5), - "xmax" => (1:10, 5), + "N" => (100:100:2000, 1000), "noise" => (0.1:0.1:1.0, 0.5), "xmax" => (1:10, 5) ) - @bind data_specs multi_slider(data_dict, title = "Parameters") + @bind data_specs multi_slider(data_dict, title="Parameters") end # ╔═╡ f0106aa5-b1c5-4857-af94-2711f80d25a8 begin - X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun = f) - scatter(X.x1, y, label = "Observed data") - xrange = range(-data_specs.xmax, data_specs.xmax, length = 50) - plot!( - xrange, - @.(f(xrange)), - lw = 4, - label = "Ground truth", - ls = :dash, - colour = :black, - ) + X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun=f) + scatter(X.x1, y; label="Observed data") + xrange = range(-data_specs.xmax, data_specs.xmax; length=50) + plot!(xrange, @.(f(xrange)); lw=4, label="Ground truth", ls=:dash, colour=:black) end # ╔═╡ 2fe1065e-d1b8-4e3c-930c-654f50349222 @@ -165,7 +153,7 @@ To start with, let's split our data into a training and test set: """ # ╔═╡ 3a4fe2bc-387c-4d7e-b45f-292075a01bcd -train, test = partition(eachindex(y), 0.4, 0.4, shuffle = true); +train, test = partition(eachindex(y), 0.4, 0.4; shuffle=true); # ╔═╡ a34b8c07-08e0-4a0e-a0f9-8054b41b038b md"Now let's choose a model for our regression task:" @@ -191,7 +179,7 @@ mach_raw = machine(model, X, y); md"Then we fit the machine to the training data:" # ╔═╡ aabfbbfb-7fb0-4f37-9a05-b96207636232 -MLJBase.fit!(mach_raw, rows = train, verbosity = 0); +MLJBase.fit!(mach_raw; rows=train, verbosity=0); # ╔═╡ 5506e1b5-5f2f-4972-a845-9c0434d4b31c md""" @@ -203,17 +191,10 @@ begin Xtest = MLJBase.matrix(selectrows(X, test)) ytest = y[test] ŷ = MLJBase.predict(mach_raw, Xtest) - scatter(vec(Xtest), vec(ytest), label = "Observed") + scatter(vec(Xtest), vec(ytest); label="Observed") _order = sortperm(vec(Xtest)) - plot!(vec(Xtest)[_order], vec(ŷ)[_order], lw = 4, label = "Predicted") - plot!( - xrange, - @.(f(xrange)), - lw = 2, - ls = :dash, - colour = :black, - label = "Ground truth", - ) + plot!(vec(Xtest)[_order], vec(ŷ)[_order]; lw=4, label="Predicted") + plot!(xrange, @.(f(xrange)); lw=2, ls=:dash, colour=:black, label="Ground truth") end # ╔═╡ 36eef47f-ad55-49be-ac60-7aa1cf50e61a @@ -247,7 +228,7 @@ Then we fit the machine to the data: """ # ╔═╡ 6b574688-ff3c-441a-a616-169685731883 -MLJBase.fit!(mach, rows = train, verbosity = 0); +MLJBase.fit!(mach; rows=train, verbosity=0); # ╔═╡ da6e8f90-a3f9-4d06-86ab-b0f6705bbf54 md""" @@ -257,22 +238,15 @@ Now let us look at the predictions for our test data again. The chart below show """ # ╔═╡ 797746e9-235f-4fb1-8cdb-9be295b54bbe -@bind coverage Slider(0.1:0.1:1.0, default = 0.8, show_value = true) +@bind coverage Slider(0.1:0.1:1.0, default=0.8, show_value=true) # ╔═╡ ad3e290b-c1f5-4008-81c7-a1a56ab10563 begin - _conf_model = conformal_model(model, coverage = coverage) + _conf_model = conformal_model(model; coverage=coverage) _mach = machine(_conf_model, X, y) - MLJBase.fit!(_mach, rows = train, verbosity = 0) - plot(_mach.model, _mach.fitresult, Xtest, ytest, zoom = 0, observed_lab = "Test points") - plot!( - xrange, - @.(f(xrange)), - lw = 2, - ls = :dash, - colour = :black, - label = "Ground truth", - ) + MLJBase.fit!(_mach; rows=train, verbosity=0) + plot(_mach.model, _mach.fitresult, Xtest, ytest; zoom=0, observed_lab="Test points") + plot!(xrange, @.(f(xrange)); lw=2, ls=:dash, colour=:black, label="Ground truth") end # ╔═╡ b3a88859-0442-41ff-bfea-313437042830 @@ -293,8 +267,9 @@ To verify the marginal coverage property empirically we can look at the empirica # ╔═╡ d1140af9-608a-4669-9595-aee72ffbaa46 begin - model_evaluation = - evaluate!(_mach, operation = MLJBase.predict, measure = emp_coverage, verbosity = 0) + model_evaluation = evaluate!( + _mach; operation=MLJBase.predict, measure=emp_coverage, verbosity=0 + ) println("Empirical coverage: $(round(model_evaluation.measurement[1], digits=3))") println("Coverage per fold: $(round.(model_evaluation.per_fold[1], digits=3))") end @@ -349,23 +324,16 @@ Quite cool, right? Using a single API call we are able to generate rigorous pred # ╔═╡ 824bd383-2fcb-4888-8ad1-260c85333edf @bind xmax_ood Slider( - data_specs.xmax:(data_specs.xmax+5), - default = (data_specs.xmax), - show_value = true, + (data_specs.xmax):(data_specs.xmax + 5), default=(data_specs.xmax), show_value=true ) # ╔═╡ 072cc72d-20a2-4ee9-954c-7ea70dfb8eea begin - Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun = f) - plot(_mach.model, _mach.fitresult, Xood, yood, zoom = 0, observed_lab = "Test points") - xood_range = range(-xmax_ood, xmax_ood, length = 50) + Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun=f) + plot(_mach.model, _mach.fitresult, Xood, yood; zoom=0, observed_lab="Test points") + xood_range = range(-xmax_ood, xmax_ood; length=50) plot!( - xood_range, - @.(f(xood_range)), - lw = 2, - ls = :dash, - colour = :black, - label = "Ground truth", + xood_range, @.(f(xood_range)); lw=2, ls=:dash, colour=:black, label="Ground truth" ) end diff --git a/docs/pluto/intro_dev.jl b/docs/pluto/intro_dev.jl index c991a44..ef775c5 100644 --- a/docs/pluto/intro_dev.jl +++ b/docs/pluto/intro_dev.jl @@ -53,10 +53,8 @@ Let's start by loading the necessary packages: # ╔═╡ 55a7c16b-a526-41d9-9d73-a0591ad006ce # helper functions begin - function multi_slider(vals::Dict; title = "") - + function multi_slider(vals::Dict; title="") return PlutoUI.combine() do Child - inputs = [ md""" $(_name): $( Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) @@ -70,7 +68,6 @@ begin $(inputs) """ end - end MLJFlux.reformat(X, ::Type{<:AbstractMatrix}) = X' @@ -100,7 +97,7 @@ Most machine learning workflows start with data. In this tutorial you have full # ╔═╡ 2f1c8da3-77dc-4bd7-8fa4-7669c2861aaa begin - function get_data(N = 600, xmax = 3.0, noise = 0.5; fun::Function = fun(X) = X * sin(X)) + function get_data(N=600, xmax=3.0, noise=0.5; fun::Function=fun(X) = X * sin(X)) # Inputs: d = Distributions.Uniform(-xmax, xmax) X = Float32.(rand(d, N)) @@ -130,26 +127,17 @@ The sliders below can be used to change the number of observations `N`, the maxi # ╔═╡ 931ce259-d5fb-4a56-beb8-61a69a2fc09e begin data_dict = Dict( - "N" => (100:100:2000, 1000), - "noise" => (0.1:0.1:1.0, 0.5), - "xmax" => (1:10, 5), + "N" => (100:100:2000, 1000), "noise" => (0.1:0.1:1.0, 0.5), "xmax" => (1:10, 5) ) - @bind data_specs multi_slider(data_dict, title = "Parameters") + @bind data_specs multi_slider(data_dict, title="Parameters") end # ╔═╡ f0106aa5-b1c5-4857-af94-2711f80d25a8 begin - X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun = f) - scatter(X.x1, y, label = "Observed data") - xrange = range(-data_specs.xmax, data_specs.xmax, length = 50) - plot!( - xrange, - @.(f(xrange)), - lw = 4, - label = "Ground truth", - ls = :dash, - colour = :black, - ) + X, y = get_data(data_specs.N, data_specs.xmax, data_specs.noise; fun=f) + scatter(X.x1, y; label="Observed data") + xrange = range(-data_specs.xmax, data_specs.xmax; length=50) + plot!(xrange, @.(f(xrange)); lw=4, label="Ground truth", ls=:dash, colour=:black) end # ╔═╡ 2fe1065e-d1b8-4e3c-930c-654f50349222 @@ -169,7 +157,7 @@ To start with, let's split our data into a training and test set: """ # ╔═╡ 3a4fe2bc-387c-4d7e-b45f-292075a01bcd -train, test = partition(eachindex(y), 0.4, 0.4, shuffle = true); +train, test = partition(eachindex(y), 0.4, 0.4; shuffle=true); # ╔═╡ a34b8c07-08e0-4a0e-a0f9-8054b41b038b md"Now let's choose a model for our regression task:" @@ -181,8 +169,7 @@ md"Now let's choose a model for our regression task:" begin Model = eval(tested_atomic_models[:regression][model_name]) if Model() isa MLJFlux.MLJFluxModel - model = - Model(builder = MLJFlux.MLP(hidden = (50,), σ = Flux.tanh_fast), epochs = 200) + model = Model(; builder=MLJFlux.MLP(; hidden=(50,), σ=Flux.tanh_fast), epochs=200) else model = Model() end @@ -200,7 +187,7 @@ mach_raw = machine(model, X, y); md"Then we fit the machine to the training data:" # ╔═╡ aabfbbfb-7fb0-4f37-9a05-b96207636232 -MLJBase.fit!(mach_raw, rows = train, verbosity = 0); +MLJBase.fit!(mach_raw; rows=train, verbosity=0); # ╔═╡ 5506e1b5-5f2f-4972-a845-9c0434d4b31c md""" @@ -212,17 +199,10 @@ begin Xtest = MLJBase.matrix(selectrows(X, test)) ytest = y[test] ŷ = MLJBase.predict(mach_raw, Xtest) - scatter(vec(Xtest), vec(ytest), label = "Observed") + scatter(vec(Xtest), vec(ytest); label="Observed") _order = sortperm(vec(Xtest)) - plot!(vec(Xtest)[_order], vec(ŷ)[_order], lw = 4, label = "Predicted") - plot!( - xrange, - @.(f(xrange)), - lw = 2, - ls = :dash, - colour = :black, - label = "Ground truth", - ) + plot!(vec(Xtest)[_order], vec(ŷ)[_order]; lw=4, label="Predicted") + plot!(xrange, @.(f(xrange)); lw=2, ls=:dash, colour=:black, label="Ground truth") end # ╔═╡ 36eef47f-ad55-49be-ac60-7aa1cf50e61a @@ -256,7 +236,7 @@ Then we fit the machine to the data: """ # ╔═╡ 6b574688-ff3c-441a-a616-169685731883 -MLJBase.fit!(mach, rows = train, verbosity = 0); +MLJBase.fit!(mach; rows=train, verbosity=0); # ╔═╡ da6e8f90-a3f9-4d06-86ab-b0f6705bbf54 md""" @@ -266,22 +246,15 @@ Now let us look at the predictions for our test data again. The chart below show """ # ╔═╡ 797746e9-235f-4fb1-8cdb-9be295b54bbe -@bind coverage Slider(0.1:0.1:1.0, default = 0.8, show_value = true) +@bind coverage Slider(0.1:0.1:1.0, default=0.8, show_value=true) # ╔═╡ ad3e290b-c1f5-4008-81c7-a1a56ab10563 begin - _conf_model = conformal_model(model, coverage = coverage) + _conf_model = conformal_model(model; coverage=coverage) _mach = machine(_conf_model, X, y) - MLJBase.fit!(_mach, rows = train, verbosity = 0) - plot(_mach.model, _mach.fitresult, Xtest, ytest, zoom = 0, observed_lab = "Test points") - plot!( - xrange, - @.(f(xrange)), - lw = 2, - ls = :dash, - colour = :black, - label = "Ground truth", - ) + MLJBase.fit!(_mach; rows=train, verbosity=0) + plot(_mach.model, _mach.fitresult, Xtest, ytest; zoom=0, observed_lab="Test points") + plot!(xrange, @.(f(xrange)); lw=2, ls=:dash, colour=:black, label="Ground truth") end # ╔═╡ b3a88859-0442-41ff-bfea-313437042830 @@ -302,8 +275,9 @@ To verify the marginal coverage property empirically we can look at the empirica # ╔═╡ d1140af9-608a-4669-9595-aee72ffbaa46 begin - model_evaluation = - evaluate!(_mach, operation = MLJBase.predict, measure = emp_coverage, verbosity = 0) + model_evaluation = evaluate!( + _mach; operation=MLJBase.predict, measure=emp_coverage, verbosity=0 + ) println("Empirical coverage: $(round(model_evaluation.measurement[1], digits=3))") println("Coverage per fold: $(round.(model_evaluation.per_fold[1], digits=3))") end @@ -358,23 +332,16 @@ Quite cool, right? Using a single API call we are able to generate rigorous pred # ╔═╡ 824bd383-2fcb-4888-8ad1-260c85333edf @bind xmax_ood Slider( - data_specs.xmax:(data_specs.xmax+5), - default = (data_specs.xmax), - show_value = true, + (data_specs.xmax):(data_specs.xmax + 5), default=(data_specs.xmax), show_value=true ) # ╔═╡ 072cc72d-20a2-4ee9-954c-7ea70dfb8eea begin - Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun = f) - plot(_mach.model, _mach.fitresult, Xood, yood, zoom = 0, observed_lab = "Test points") - xood_range = range(-xmax_ood, xmax_ood, length = 50) + Xood, yood = get_data(data_specs.N, xmax_ood, data_specs.noise; fun=f) + plot(_mach.model, _mach.fitresult, Xood, yood; zoom=0, observed_lab="Test points") + xood_range = range(-xmax_ood, xmax_ood; length=50) plot!( - xood_range, - @.(f(xood_range)), - lw = 2, - ls = :dash, - colour = :black, - label = "Ground truth", + xood_range, @.(f(xood_range)); lw=2, ls=:dash, colour=:black, label="Ground truth" ) end diff --git a/docs/pluto/understanding_coverage.jl b/docs/pluto/understanding_coverage.jl index 498f117..f4fff33 100644 --- a/docs/pluto/understanding_coverage.jl +++ b/docs/pluto/understanding_coverage.jl @@ -45,10 +45,8 @@ md""" # ╔═╡ 59bdf205-b1d3-42ca-a256-f2878a87a691 # helper functions begin - function multi_slider(vals::Dict; title = "") - + function multi_slider(vals::Dict; title="") return PlutoUI.combine() do Child - inputs = [ md""" $(_name): $( Child(_name, Slider(_vals[1], default=_vals[2], show_value=true)) @@ -62,7 +60,6 @@ begin $(inputs) """ end - end end; @@ -76,10 +73,10 @@ We will use a simple helper function that generates some regression data with a # ╔═╡ 3dfc7d81-0644-4892-ba0d-7f3baba37ece begin function get_data( - N = 600, - noise = 0.5; - fun::Function = fun(X) = X * sin(X), - d::Distribution = Distributions.Normal(0, 1), + N=600, + noise=0.5; + fun::Function=fun(X) = X * sin(X), + d::Distribution=Distributions.Normal(0, 1), ) # Inputs: X = rand(d, N) @@ -109,12 +106,9 @@ Below you can select the type of distribution that generates our input $X$: # ╔═╡ 39c53541-356a-4051-891e-4342887fd423 begin dist_dict = Dict( - "Cauchy" => Cauchy, - "Cosine" => Cosine, - "Laplace" => Laplace, - "Normal" => Normal, + "Cauchy" => Cauchy, "Cosine" => Cosine, "Laplace" => Laplace, "Normal" => Normal ) - @bind dist Select(collect(keys(dist_dict)), default = "Normal") + @bind dist Select(collect(keys(dist_dict)), default="Normal") end # ╔═╡ 6afc2fe2-e4ac-43d0-a4e5-ea4cd7ce82fc @@ -132,24 +126,24 @@ begin "location" => (-10000:1000:10000, 0), "scale" => ([1, 2, 5, 10, 50, 100, 1000], 1), ) - @bind data_specs multi_slider(data_dict, title = "Parameters") + @bind data_specs multi_slider(data_dict, title="Parameters") end # ╔═╡ f5bbf50c-21d6-4aad-8d8f-781246382131 begin d = dist_dict[dist](data_specs.location, data_specs.scale) - X, y = get_data(data_specs.N, data_specs.noise; fun = f, d = d) - train, test = partition(eachindex(y), 0.4, 0.4, shuffle = true) - scatter(X.x1, y, label = "Observed data") - xrange = range(minimum(X.x1)[1], maximum(X.x1)[1], length = 50) + X, y = get_data(data_specs.N, data_specs.noise; fun=f, d=d) + train, test = partition(eachindex(y), 0.4, 0.4; shuffle=true) + scatter(X.x1, y; label="Observed data") + xrange = range(minimum(X.x1)[1], maximum(X.x1)[1]; length=50) plot!( xrange, - @.(f(xrange)), - lw = 4, - label = "Ground truth", - ls = :dash, - colour = :black, - size = (800, 200), + @.(f(xrange)); + lw=4, + label="Ground truth", + ls=:dash, + colour=:black, + size=(800, 200), ) end @@ -168,29 +162,29 @@ begin end; # ╔═╡ e6feb79d-f524-4d06-b18b-51ee6854c30b -@bind coverage Slider(0.1:0.01:1.0, default = 0.8, show_value = true) +@bind coverage Slider(0.1:0.01:1.0, default=0.8, show_value=true) # ╔═╡ bb75d0a7-c0b4-484c-b3da-224e368b743a begin Xtest = MLJBase.matrix(selectrows(X, test)) ytest = y[test] - conf_model = conformal_model(model, coverage = coverage) + conf_model = conformal_model(model; coverage=coverage) mach = machine(conf_model, X, y) - MLJBase.fit!(mach, rows = train, verbosity = 0) + MLJBase.fit!(mach; rows=train, verbosity=0) s = conf_model.scores α = 1 - conf_model.coverage scatter( s, zeros(length(s)); - ylim = (-0.5, 0.5), - yaxis = nothing, - label = "Scores", - ms = 10, - alpha = 0.2, - size = (800, 150), + ylim=(-0.5, 0.5), + yaxis=nothing, + label="Scores", + ms=10, + alpha=0.2, + size=(800, 150), ) q̂ = quantile(s, 1 - α) - vline!([q̂], label = "q̂") + vline!([q̂]; label="q̂") end # ╔═╡ 00000000-0000-0000-0000-000000000001 diff --git a/docs/setup_docs.jl b/docs/setup_docs.jl index f12a6e6..c45f9f5 100644 --- a/docs/setup_docs.jl +++ b/docs/setup_docs.jl @@ -22,11 +22,10 @@ setup_docs = quote # Explicit imports: import MLJModelInterface as MMI - import UnicodePlots + using UnicodePlots: UnicodePlots # Setup: theme(:wong) Random.seed!(2023) www_path = "$(pwd())/docs/src/www" - end; diff --git a/src/conformal_models/conformal_models.jl b/src/conformal_models/conformal_models.jl index cd14825..1a7d291 100644 --- a/src/conformal_models/conformal_models.jl +++ b/src/conformal_models/conformal_models.jl @@ -1,7 +1,7 @@ using MLJBase import MLJModelInterface as MMI import MLJModelInterface: predict, fit, save, restore -import StatsBase +using StatsBase: StatsBase "An abstract base type for conformal models that produce interval-valued predictions. This includes most conformal regression models." abstract type ConformalInterval <: MMI.Interval end @@ -12,8 +12,9 @@ abstract type ConformalProbabilisticSet <: MMI.ProbabilisticSet end "An abstract base type for conformal models that produce probabilistic predictions. This includes some conformal classifier like Venn-ABERS." abstract type ConformalProbabilistic <: MMI.Probabilistic end -const ConformalModel = - Union{ConformalInterval,ConformalProbabilisticSet,ConformalProbabilistic} +const ConformalModel = Union{ + ConformalInterval,ConformalProbabilisticSet,ConformalProbabilistic +} include("utils.jl") include("plotting.jl") @@ -25,11 +26,8 @@ include("plotting.jl") A simple wrapper function that turns a `model::Supervised` into a conformal model. It accepts an optional key argument that can be used to specify the desired `method` for conformal prediction as well as additinal `kwargs...` specific to the `method`. """ function conformal_model( - model::Supervised; - method::Union{Nothing,Symbol} = nothing, - kwargs..., + model::Supervised; method::Union{Nothing,Symbol}=nothing, kwargs... ) - is_classifier = target_scitype(model) <: AbstractVector{<:Finite} if isnothing(method) @@ -49,7 +47,6 @@ function conformal_model( conf_model = _method(model; kwargs...) return conf_model - end # Regression Models: @@ -64,8 +61,9 @@ include("transductive_classification.jl") include("training/training.jl") # Type unions: -const InductiveModel = - Union{SimpleInductiveRegressor,SimpleInductiveClassifier,AdaptiveInductiveClassifier} +const InductiveModel = Union{ + SimpleInductiveRegressor,SimpleInductiveClassifier,AdaptiveInductiveClassifier +} const TransductiveModel = Union{ NaiveRegressor, diff --git a/src/conformal_models/inductive_classification.jl b/src/conformal_models/inductive_classification.jl index 031f03b..7fd0d51 100644 --- a/src/conformal_models/inductive_classification.jl +++ b/src/conformal_models/inductive_classification.jl @@ -7,9 +7,9 @@ function score( conf_model::ConformalProbabilisticSet, fitresult, X, - y::Union{Nothing,AbstractArray} = nothing, + y::Union{Nothing,AbstractArray}=nothing, ) - score(conf_model, typeof(conf_model.model), fitresult, X, y) + return score(conf_model, typeof(conf_model.model), fitresult, X, y) end """ @@ -40,9 +40,9 @@ end function SimpleInductiveClassifier( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(p̂) = 1.0 - p̂, - train_ratio::AbstractFloat = 0.5, + coverage::AbstractFloat=0.95, + heuristic::Function=f(p̂) = 1.0 - p̂, + train_ratio::AbstractFloat=0.5, ) return SimpleInductiveClassifier(model, coverage, nothing, heuristic, train_ratio) end @@ -57,7 +57,7 @@ function score( ::Type{<:Supervised}, fitresult, X, - y::Union{Nothing,AbstractArray} = nothing, + y::Union{Nothing,AbstractArray}=nothing, ) p̂ = reformat_mlj_prediction(MMI.predict(conf_model.model, fitresult, X)) L = p̂.decoder.classes @@ -110,7 +110,7 @@ where ``\mathcal{D}_{\text{calibration}}`` denotes the designated calibration da """ function MMI.predict(conf_model::SimpleInductiveClassifier, fitresult, Xnew) p̂ = reformat_mlj_prediction( - MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...) ) v = conf_model.scores[:calibration] q̂ = StatsBase.quantile(v, conf_model.coverage) @@ -140,9 +140,9 @@ end function AdaptiveInductiveClassifier( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = 1.0 - ŷ, - train_ratio::AbstractFloat = 0.5, + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = 1.0 - ŷ, + train_ratio::AbstractFloat=0.5, ) return AdaptiveInductiveClassifier(model, coverage, nothing, heuristic, train_ratio) end @@ -181,7 +181,7 @@ function score( ::Type{<:Supervised}, fitresult, X, - y::Union{Nothing,AbstractArray} = nothing, + y::Union{Nothing,AbstractArray}=nothing, ) p̂ = reformat_mlj_prediction(MMI.predict(conf_model.model, fitresult, X)) L = p̂.decoder.classes @@ -214,7 +214,7 @@ where ``\mathcal{D}_{\text{calibration}}`` denotes the designated calibration da """ function MMI.predict(conf_model::AdaptiveInductiveClassifier, fitresult, Xnew) p̂ = reformat_mlj_prediction( - MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...) ) v = conf_model.scores[:calibration] q̂ = StatsBase.quantile(v, conf_model.coverage) diff --git a/src/conformal_models/inductive_regression.jl b/src/conformal_models/inductive_regression.jl index 3afaa40..c9d11c9 100644 --- a/src/conformal_models/inductive_regression.jl +++ b/src/conformal_models/inductive_regression.jl @@ -9,9 +9,9 @@ end function SimpleInductiveRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), - train_ratio::AbstractFloat = 0.5, + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), + train_ratio::AbstractFloat=0.5, ) return SimpleInductiveRegressor(model, coverage, nothing, heuristic, train_ratio) end @@ -62,7 +62,7 @@ where ``\mathcal{D}_{\text{calibration}}`` denotes the designated calibration da """ function MMI.predict(conf_model::SimpleInductiveRegressor, fitresult, Xnew) ŷ = reformat_mlj_prediction( - MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...) ) v = conf_model.scores q̂ = StatsBase.quantile(v, conf_model.coverage) diff --git a/src/conformal_models/plotting.jl b/src/conformal_models/plotting.jl index f7c4f12..e02862b 100644 --- a/src/conformal_models/plotting.jl +++ b/src/conformal_models/plotting.jl @@ -59,17 +59,17 @@ function Plots.contourf( fitresult, X, y; - target::Union{Nothing,Real} = nothing, - ntest = 50, - zoom = -1, - xlims = nothing, - ylims = nothing, - plot_set_size = false, - plot_classification_loss = false, - plot_set_loss = false, - temp = nothing, - κ = 0, - loss_matrix = UniformScaling(1.0), + target::Union{Nothing,Real}=nothing, + ntest=50, + zoom=-1, + xlims=nothing, + ylims=nothing, + plot_set_size=false, + plot_classification_loss=false, + plot_set_loss=false, + temp=nothing, + κ=0, + loss_matrix=UniformScaling(1.0), kwargs..., ) @@ -85,8 +85,8 @@ function Plots.contourf( xlims, ylims = generate_lims(x1, x2, xlims, ylims, zoom) # Surface range: - x1range = range(xlims[1], stop = xlims[2], length = ntest) - x2range = range(ylims[1], stop = ylims[2], length = ntest) + x1range = range(xlims[1]; stop=xlims[2], length=ntest) + x2range = range(ylims[1]; stop=ylims[2], length=ntest) # Target if !isnothing(target) @@ -126,25 +126,16 @@ function Plots.contourf( if plot_set_size z = ismissing(p̂) ? 0 : sum(pdf.(p̂, p̂.decoder.classes) .> 0) elseif plot_classification_loss - _target = categorical([target], levels = levels(y)) + _target = categorical([target]; levels=levels(y)) z = ConformalPrediction.classification_loss( - conf_model, - fitresult, - [x1 x2], - _target; - temp = temp, - loss_matrix = loss_matrix, + conf_model, fitresult, [x1 x2], _target; temp=temp, loss_matrix=loss_matrix ) elseif plot_set_loss z = ConformalPrediction.smooth_size_loss( - conf_model, - fitresult, - [x1 x2]; - κ = κ, - temp = temp, + conf_model, fitresult, [x1 x2]; κ=κ, temp=temp ) else - z = ismissing(p̂) ? [missing for i = 1:length(levels(y))] : pdf.(p̂, levels(y)) + z = ismissing(p̂) ? [missing for i in 1:length(levels(y))] : pdf.(p̂, levels(y)) z = replace(z, 0 => missing) end push!(Z, z) @@ -160,11 +151,11 @@ function Plots.contourf( x1range, x2range, Z; - title = title, - xlims = xlims, - ylims = ylims, - c = cgrad(:blues, _n + 1, categorical = true), - clim = clim, + title=title, + xlims=xlims, + ylims=ylims, + c=cgrad(:blues, _n + 1; categorical=true), + clim=clim, kwargs..., ) else @@ -173,20 +164,19 @@ function Plots.contourf( x1range, x2range, Z; - title = title, - xlims = xlims, - ylims = ylims, - clim = clim, - c = cgrad(:blues), - linewidth = 0, + title=title, + xlims=xlims, + ylims=ylims, + clim=clim, + c=cgrad(:blues), + linewidth=0, kwargs..., ) end # Samples: y = typeof(y) <: CategoricalArrays.CategoricalArray ? y : Int.(y) - scatter!(plt, x1, x2, group = y; kwargs...) - + return scatter!(plt, x1, x2; group=y, kwargs...) end """ @@ -203,7 +193,7 @@ function Plots.areaplot( fitresult, X, y; - input_var::Union{Nothing,Int,Symbol} = nothing, + input_var::Union{Nothing,Int,Symbol}=nothing, kwargs..., ) @@ -235,11 +225,10 @@ function Plots.areaplot( ŷ = predict(conf_model, fitresult, Xraw) nout = length(levels(y)) ŷ = - map(_y -> ismissing(_y) ? [0 for i = 1:nout] : pdf.(_y, levels(y)), ŷ) |> _y -> reduce(hcat, _y) + map(_y -> ismissing(_y) ? [0 for i in 1:nout] : pdf.(_y, levels(y)), ŷ) |> _y -> reduce(hcat, _y) ŷ = permutedims(ŷ) - areaplot(x, ŷ; kwargs...) - + return areaplot(x, ŷ; kwargs...) end """ @@ -255,13 +244,13 @@ function Plots.plot( fitresult, X, y; - input_var::Union{Nothing,Int,Symbol} = nothing, - xlims::Union{Nothing,Tuple} = nothing, - ylims::Union{Nothing,Tuple} = nothing, - zoom::Real = -0.5, - train_lab::Union{Nothing,String} = nothing, - test_lab::Union{Nothing,String} = nothing, - ymid_lw::Int = 1, + input_var::Union{Nothing,Int,Symbol}=nothing, + xlims::Union{Nothing,Tuple}=nothing, + ylims::Union{Nothing,Tuple}=nothing, + zoom::Real=-0.5, + train_lab::Union{Nothing,String}=nothing, + test_lab::Union{Nothing,String}=nothing, + ymid_lw::Int=1, kwargs..., ) @@ -298,13 +287,7 @@ function Plots.plot( # Plot training data: plt = scatter( - vec(x), - vec(y), - label = train_lab, - xlim = xlims, - ylim = ylims, - title = title; - kwargs..., + vec(x), vec(y); label=train_lab, xlim=xlims, ylim=ylims, title=title, kwargs... ) # Plot predictions: @@ -314,16 +297,15 @@ function Plots.plot( yerror = (ub .- lb) ./ 2 xplot = vec(x) _idx = sortperm(xplot) - plot!( + return plot!( plt, xplot[_idx], - ymid[_idx], - label = test_lab, - ribbon = (yerror, yerror), - lw = ymid_lw; + ymid[_idx]; + label=test_lab, + ribbon=(yerror, yerror), + lw=ymid_lw, kwargs..., ) - end """ @@ -332,16 +314,11 @@ end A `Plots.jl` recipe/method extension that can be used to visualize the set size distribution of a conformal predictor. In the regression case, prediction interval widths are stratified into discrete bins. It can be useful to plot the distribution of set sizes in order to visually asses how adaptive a conformal predictor is. For more adaptive predictors the distribution of set sizes is typically spread out more widely, which reflects that “the procedure is effectively distinguishing between easy and hard inputs”. This is desirable: when for a given sample it is difficult to make predictions, this should be reflected in the set size (or interval width in the regression case). Since ‘difficult’ lies on some spectrum that ranges from ‘very easy’ to ‘very difficult’ the set size should very across the spectrum of ‘empty set’ to ‘all labels included’. """ function Plots.bar( - conf_model::ConformalModel, - fitresult, - X; - label = "", - xtickfontsize = 6, - kwrgs..., + conf_model::ConformalModel, fitresult, X; label="", xtickfontsize=6, kwrgs... ) ŷ = predict(conf_model, fitresult, X) idx = size_indicator(ŷ) - x = sort(levels(idx), lt = natural) + x = sort(levels(idx); lt=natural) y = [sum(idx .== _x) for _x in x] - Plots.bar(x, y; label = label, xtickfontsize = xtickfontsize, kwrgs...) + return Plots.bar(x, y; label=label, xtickfontsize=xtickfontsize, kwrgs...) end diff --git a/src/conformal_models/training/inductive_classification.jl b/src/conformal_models/training/inductive_classification.jl index 589d373..990cf78 100644 --- a/src/conformal_models/training/inductive_classification.jl +++ b/src/conformal_models/training/inductive_classification.jl @@ -12,7 +12,7 @@ function score( ::Type{<:MLJFluxModel}, fitresult, X, - y::Union{Nothing,AbstractArray} = nothing, + y::Union{Nothing,AbstractArray}=nothing, ) X = reformat(X) X = typeof(X) <: AbstractArray ? X : permutedims(matrix(X)) @@ -36,7 +36,7 @@ function score( ::Type{<:EitherEnsembleModel{<:MLJFluxModel}}, fitresult, X, - y::Union{Nothing,AbstractArray} = nothing, + y::Union{Nothing,AbstractArray}=nothing, ) X = reformat(X) X = typeof(X) <: AbstractArray ? X : permutedims(matrix(X)) @@ -44,8 +44,8 @@ function score( probas = MLUtils.stack(map(chain -> chain(X), _chains)) |> p -> - mean(p, dims = ndims(p)) |> - p -> MLUtils.unstack(p, dims = ndims(p))[1] |> p -> permutedims(p) + mean(p; dims=ndims(p)) |> + p -> MLUtils.unstack(p; dims=ndims(p))[1] |> p -> permutedims(p) scores = @.(conf_model.heuristic(probas)) if isnothing(y) return scores @@ -65,7 +65,7 @@ function score( ::Type{<:MLJFluxModel}, fitresult, X, - y::Union{Nothing,AbstractArray} = nothing, + y::Union{Nothing,AbstractArray}=nothing, ) L = levels(fitresult[2]) X = reformat(X) @@ -96,7 +96,7 @@ function score( ::Type{<:EitherEnsembleModel{<:MLJFluxModel}}, fitresult, X, - y::Union{Nothing,AbstractArray} = nothing, + y::Union{Nothing,AbstractArray}=nothing, ) L = levels(fitresult.ensemble[1][2]) X = reformat(X) @@ -105,8 +105,8 @@ function score( probas = MLUtils.stack(map(chain -> chain(X), _chains)) |> p -> - mean(p, dims = ndims(p)) |> - p -> MLUtils.unstack(p, dims = ndims(p))[1] |> p -> permutedims(p) + mean(p; dims=ndims(p)) |> + p -> MLUtils.unstack(p; dims=ndims(p))[1] |> p -> permutedims(p) scores = map(Base.Iterators.product(eachrow(probas), L)) do Z probasᵢ, yₖ = Z ranks = sortperm(.-probasᵢ) # rank in descending order diff --git a/src/conformal_models/training/losses.jl b/src/conformal_models/training/losses.jl index 2b69045..5efad65 100644 --- a/src/conformal_models/training/losses.jl +++ b/src/conformal_models/training/losses.jl @@ -8,12 +8,11 @@ using MLJBase Computes soft assignment scores for each label and sample. That is, the probability of label `k` being included in the confidence set. This implementation follows Stutz et al. (2022): https://openreview.net/pdf?id=t8O-4LKFVx. Contrary to the paper, we use non-conformity scores instead of conformity scores, hence the sign swap. """ function soft_assignment( - conf_model::ConformalProbabilisticSet; - temp::Union{Nothing,Real} = nothing, + conf_model::ConformalProbabilisticSet; temp::Union{Nothing,Real}=nothing ) temp = isnothing(temp) ? 0.5 : temp v = sort(conf_model.scores[:calibration]) - q̂ = StatsBase.quantile(v, conf_model.coverage, sorted = true) + q̂ = StatsBase.quantile(v, conf_model.coverage; sorted=true) scores = conf_model.scores[:all] return @.(σ((q̂ - scores) / temp)) end @@ -24,14 +23,11 @@ end This function can be used to compute soft assigment probabilities for new data `X` as in [`soft_assignment(conf_model::ConformalProbabilisticSet; temp::Real=0.5)`](@ref). When a fitted model $\mu$ (`fitresult`) and new samples `X` are supplied, non-conformity scores are first computed for the new data points. Then the existing threshold/quantile `q̂` is used to compute the final soft assignments. """ function soft_assignment( - conf_model::ConformalProbabilisticSet, - fitresult, - X; - temp::Union{Nothing,Real} = nothing, + conf_model::ConformalProbabilisticSet, fitresult, X; temp::Union{Nothing,Real}=nothing ) temp = isnothing(temp) ? 0.5 : temp v = sort(conf_model.scores[:calibration]) - q̂ = StatsBase.quantile(v, conf_model.coverage, sorted = true) + q̂ = StatsBase.quantile(v, conf_model.coverage; sorted=true) scores = score(conf_model, fitresult, X) return @.(σ((q̂ - scores) / temp)) end @@ -54,13 +50,14 @@ function smooth_size_loss( conf_model::ConformalProbabilisticSet, fitresult, X; - temp::Union{Nothing,Real} = nothing, - κ::Real = 1.0, + temp::Union{Nothing,Real}=nothing, + κ::Real=1.0, ) temp = isnothing(temp) ? 0.5 : temp - C = soft_assignment(conf_model, fitresult, X; temp = temp) - is_empty_set = - all(x -> x .== 0, soft_assignment(conf_model, fitresult, X; temp = 0.0), dims = 2) + C = soft_assignment(conf_model, fitresult, X; temp=temp) + is_empty_set = all( + x -> x .== 0, soft_assignment(conf_model, fitresult, X; temp=0.0); dims=2 + ) Ω = [] for i in eachindex(is_empty_set) c = C[i, :] @@ -104,8 +101,8 @@ function classification_loss( fitresult, X, y; - loss_matrix::Union{AbstractMatrix,UniformScaling} = UniformScaling(1.0), - temp::Union{Nothing,Real} = nothing, + loss_matrix::Union{AbstractMatrix,UniformScaling}=UniformScaling(1.0), + temp::Union{Nothing,Real}=nothing, ) # Setup: temp = isnothing(temp) ? 0.5 : temp @@ -119,7 +116,7 @@ function classification_loss( if typeof(loss_matrix) <: UniformScaling loss_matrix = Matrix(loss_matrix(K)) end - C = soft_assignment(conf_model, fitresult, X; temp = temp) + C = soft_assignment(conf_model, fitresult, X; temp=temp) # Loss: ℒ = map(eachrow(C), eachrow(yenc)) do c, _yenc diff --git a/src/conformal_models/transductive_classification.jl b/src/conformal_models/transductive_classification.jl index 9227dd5..4c564e0 100644 --- a/src/conformal_models/transductive_classification.jl +++ b/src/conformal_models/transductive_classification.jl @@ -8,9 +8,7 @@ mutable struct NaiveClassifier{Model<:Supervised} <: ConformalProbabilisticSet end function NaiveClassifier( - model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = 1.0 - ŷ, + model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=f(y, ŷ) = 1.0 - ŷ ) return NaiveClassifier(model, coverage, nothing, heuristic) end @@ -45,7 +43,6 @@ function MMI.fit(conf_model::NaiveClassifier, verbosity, X, y) conf_model.scores = @.(conf_model.heuristic(y, ŷ)) return (fitresult, cache, report) - end @doc raw""" @@ -61,7 +58,7 @@ The naive approach typically produces prediction regions that undercover due to """ function MMI.predict(conf_model::NaiveClassifier, fitresult, Xnew) p̂ = reformat_mlj_prediction( - MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...) ) v = conf_model.scores q̂ = StatsBase.quantile(v, conf_model.coverage) diff --git a/src/conformal_models/transductive_regression.jl b/src/conformal_models/transductive_regression.jl index dfaf361..aa78a3f 100644 --- a/src/conformal_models/transductive_regression.jl +++ b/src/conformal_models/transductive_regression.jl @@ -14,8 +14,8 @@ end function NaiveRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), ) return NaiveRegressor(model, coverage, nothing, heuristic) end @@ -46,7 +46,6 @@ function MMI.fit(conf_model::NaiveRegressor, verbosity, X, y) conf_model.scores = @.(conf_model.heuristic(ytrain, ŷ)) return (fitresult, cache, report) - end # Prediction @@ -63,7 +62,7 @@ The naive approach typically produces prediction regions that undercover due to """ function MMI.predict(conf_model::NaiveRegressor, fitresult, Xnew) ŷ = reformat_mlj_prediction( - MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...) ) v = conf_model.scores q̂ = StatsBase.quantile(v, conf_model.coverage) @@ -83,8 +82,8 @@ end function JackknifeRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), ) return JackknifeRegressor(model, coverage, nothing, heuristic) end @@ -114,7 +113,7 @@ function MMI.fit(conf_model::JackknifeRegressor, verbosity, X, y) # Nonconformity Scores: T = size(y, 1) scores = [] - for t = 1:T + for t in 1:T loo_ids = 1:T .!= t y₋ᵢ = y[loo_ids] X₋ᵢ = selectrows(X, loo_ids) @@ -122,7 +121,7 @@ function MMI.fit(conf_model::JackknifeRegressor, verbosity, X, y) Xᵢ = selectrows(X, t) μ̂₋ᵢ, = MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, X₋ᵢ, y₋ᵢ)...) ŷᵢ = reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xᵢ)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xᵢ)...) ) push!(scores, @.(conf_model.heuristic(yᵢ, ŷᵢ))...) end @@ -145,7 +144,7 @@ where ``S_i^{\text{LOO}}`` denotes the nonconformity that is generated as explai """ function MMI.predict(conf_model::JackknifeRegressor, fitresult, Xnew) ŷ = reformat_mlj_prediction( - MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xnew)...) ) v = conf_model.scores q̂ = StatsBase.quantile(v, conf_model.coverage) @@ -165,8 +164,8 @@ end function JackknifePlusRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), ) return JackknifePlusRegressor(model, coverage, nothing, heuristic) end @@ -190,21 +189,22 @@ function MMI.fit(conf_model::JackknifePlusRegressor, verbosity, X, y) # Nonconformity Scores: T = size(y, 1) scores = [] - for t = 1:T + for t in 1:T loo_ids = 1:T .!= t y₋ᵢ = y[loo_ids] X₋ᵢ = selectrows(X, loo_ids) yᵢ = y[t] Xᵢ = selectrows(X, t) # Store LOO fitresult: - μ̂₋ᵢ, cache₋ᵢ, report₋ᵢ = - MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, X₋ᵢ, y₋ᵢ)...) + μ̂₋ᵢ, cache₋ᵢ, report₋ᵢ = MMI.fit( + conf_model.model, 0, MMI.reformat(conf_model.model, X₋ᵢ, y₋ᵢ)... + ) push!(fitresult, μ̂₋ᵢ) push!(cache, cache₋ᵢ) push!(report, report₋ᵢ) # Store LOO score: ŷᵢ = reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xᵢ)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xᵢ)...) ) push!(scores, @.(conf_model.heuristic(yᵢ, ŷᵢ))...) end @@ -229,7 +229,7 @@ function MMI.predict(conf_model::JackknifePlusRegressor, fitresult, Xnew) # Get all LOO predictions for each Xnew: ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...) ) for μ̂₋ᵢ in fitresult ] # All LOO predictions across columns for each Xnew across rows: @@ -255,8 +255,8 @@ end function JackknifeMinMaxRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), ) return JackknifeMinMaxRegressor(model, coverage, nothing, heuristic) end @@ -280,21 +280,22 @@ function MMI.fit(conf_model::JackknifeMinMaxRegressor, verbosity, X, y) # Training and Nonconformity Scores: T = size(y, 1) scores = [] - for t = 1:T + for t in 1:T loo_ids = 1:T .!= t y₋ᵢ = y[loo_ids] X₋ᵢ = selectrows(X, loo_ids) yᵢ = y[t] Xᵢ = selectrows(X, t) # Store LOO fitresult: - μ̂₋ᵢ, cache₋ᵢ, report₋ᵢ = - MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, X₋ᵢ, y₋ᵢ)...) + μ̂₋ᵢ, cache₋ᵢ, report₋ᵢ = MMI.fit( + conf_model.model, 0, MMI.reformat(conf_model.model, X₋ᵢ, y₋ᵢ)... + ) push!(fitresult, μ̂₋ᵢ) push!(cache, cache₋ᵢ) push!(report, report₋ᵢ) # Store LOO score: ŷᵢ = reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xᵢ)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xᵢ)...) ) push!(scores, @.(conf_model.heuristic(yᵢ, ŷᵢ))...) end @@ -319,7 +320,7 @@ function MMI.predict(conf_model::JackknifeMinMaxRegressor, fitresult, Xnew) # Get all LOO predictions for each Xnew: ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...) ) for μ̂₋ᵢ in fitresult ] # All LOO predictions across columns for each Xnew across rows: @@ -345,9 +346,9 @@ end function CVPlusRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), - cv::MLJBase.CV = MLJBase.CV(), + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), + cv::MLJBase.CV=MLJBase.CV(), ) return CVPlusRegressor(model, coverage, nothing, heuristic, cv) end @@ -372,9 +373,7 @@ function MMI.fit(conf_model::CVPlusRegressor, verbosity, X, y) ytrain = y[train] Xtrain = selectrows(X, train) μ̂ₖ, cache, report = MMI.fit( - conf_model.model, - 0, - MMI.reformat(conf_model.model, Xtrain, ytrain)..., + conf_model.model, 0, MMI.reformat(conf_model.model, Xtrain, ytrain)... ) Dict(:fitresult => μ̂ₖ, :test => test, :cache => cache, :report => report) end @@ -384,11 +383,12 @@ function MMI.fit(conf_model::CVPlusRegressor, verbosity, X, y) # Nonconformity Scores: scores = [] - for t = 1:T + for t in 1:T yᵢ = y[t] Xᵢ = selectrows(X, t) - resultsᵢ = - [(x[:fitresult], x[:cache], x[:report]) for x in cv_fitted if t in x[:test]] + resultsᵢ = [ + (x[:fitresult], x[:cache], x[:report]) for x in cv_fitted if t in x[:test] + ] @assert length(resultsᵢ) == 1 "Expected each individual to be contained in only one subset." μ̂ᵢ, cacheᵢ, reportᵢ = resultsᵢ[1] # Store individual CV fitresults @@ -397,7 +397,7 @@ function MMI.fit(conf_model::CVPlusRegressor, verbosity, X, y) push!(report, reportᵢ) # Store LOO score: ŷᵢ = reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂ᵢ, MMI.reformat(conf_model.model, Xᵢ)...), + MMI.predict(conf_model.model, μ̂ᵢ, MMI.reformat(conf_model.model, Xᵢ)...) ) push!(scores, @.(conf_model.heuristic(yᵢ, ŷᵢ))...) end @@ -424,7 +424,7 @@ function MMI.predict(conf_model::CVPlusRegressor, fitresult, Xnew) # Get all LOO predictions for each Xnew: ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...) ) for μ̂₋ᵢ in fitresult ] # All LOO predictions across columns for each Xnew across rows: @@ -439,7 +439,6 @@ function MMI.predict(conf_model::CVPlusRegressor, fitresult, Xnew) return ŷ end - # CV MinMax "Constructor for `CVMinMaxRegressor`." mutable struct CVMinMaxRegressor{Model<:Supervised} <: ConformalInterval @@ -452,9 +451,9 @@ end function CVMinMaxRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), - cv::MLJBase.CV = MLJBase.CV(), + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), + cv::MLJBase.CV=MLJBase.CV(), ) return CVMinMaxRegressor(model, coverage, nothing, heuristic, cv) end @@ -479,9 +478,7 @@ function MMI.fit(conf_model::CVMinMaxRegressor, verbosity, X, y) ytrain = y[train] Xtrain = selectrows(X, train) μ̂ₖ, cache, report = MMI.fit( - conf_model.model, - 0, - MMI.reformat(conf_model.model, Xtrain, ytrain)..., + conf_model.model, 0, MMI.reformat(conf_model.model, Xtrain, ytrain)... ) Dict(:fitresult => μ̂ₖ, :test => test, :cache => cache, :report => report) end @@ -491,11 +488,12 @@ function MMI.fit(conf_model::CVMinMaxRegressor, verbosity, X, y) # Nonconformity Scores: scores = [] - for t = 1:T + for t in 1:T yᵢ = y[t] Xᵢ = selectrows(X, t) - resultsᵢ = - [(x[:fitresult], x[:cache], x[:report]) for x in cv_fitted if t in x[:test]] + resultsᵢ = [ + (x[:fitresult], x[:cache], x[:report]) for x in cv_fitted if t in x[:test] + ] @assert length(resultsᵢ) == 1 "Expected each individual to be contained in only one subset." μ̂ᵢ, cacheᵢ, reportᵢ = resultsᵢ[1] # Store individual CV fitresults @@ -504,7 +502,7 @@ function MMI.fit(conf_model::CVMinMaxRegressor, verbosity, X, y) push!(report, reportᵢ) # Store LOO score: ŷᵢ = reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂ᵢ, MMI.reformat(conf_model.model, Xᵢ)...), + MMI.predict(conf_model.model, μ̂ᵢ, MMI.reformat(conf_model.model, Xᵢ)...) ) push!(scores, @.(conf_model.heuristic(yᵢ, ŷᵢ))...) end @@ -513,7 +511,6 @@ function MMI.fit(conf_model::CVMinMaxRegressor, verbosity, X, y) return (fitresult, cache, report) end - # Prediction @doc raw""" MMI.predict(conf_model::CVMinMaxRegressor, fitresult, Xnew) @@ -530,7 +527,7 @@ function MMI.predict(conf_model::CVMinMaxRegressor, fitresult, Xnew) # Get all LOO predictions for each Xnew: ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...) ) for μ̂₋ᵢ in fitresult ] # All LOO predictions across columns for each Xnew across rows: @@ -554,7 +551,7 @@ function _aggregate(y, aggregate::Union{Symbol,String}) valid_methods = Dict( :mean => x -> StatsBase.mean(x), :median => x -> StatsBase.median(x), - :trimmedmean => x -> StatsBase.mean(trim(x, prop = 0.1)), + :trimmedmean => x -> StatsBase.mean(trim(x; prop=0.1)), ) @assert aggregate ∈ keys(valid_methods) "`aggregate`=$aggregate is not a valid aggregation method. Should be one of: $valid_methods" # Aggregate: @@ -582,22 +579,15 @@ end function JackknifePlusAbRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), - nsampling::Int = 30, - sample_size::AbstractFloat = 0.5, - replacement::Bool = true, - aggregate::Union{Symbol,String} = "mean", + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), + nsampling::Int=30, + sample_size::AbstractFloat=0.5, + replacement::Bool=true, + aggregate::Union{Symbol,String}="mean", ) return JackknifePlusAbRegressor( - model, - coverage, - nothing, - heuristic, - nsampling, - sample_size, - replacement, - aggregate, + model, coverage, nothing, heuristic, nsampling, sample_size, replacement, aggregate ) end @@ -613,7 +603,6 @@ $ S_i^{\text{J+ab}} = s(X_i, Y_i) = h(agg(\hat\mu_{B_{K(-i)}}(X_i)), Y_i), \ i \ where ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` denotes the aggregate predictions, typically mean or median, for each ``X_i`` (with ``K_{-i}`` the bootstraps not containing ``X_i``). In other words, B models are trained on boostrapped sampling, the fitted models are then used to create aggregated prediction of out-of-sample ``X_i``. The corresponding nonconformity score is then computed by applying a heuristic uncertainty measure ``h(\cdot)`` to the fitted value ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` and the true value ``Y_i``. """ function MMI.fit(conf_model::JackknifePlusAbRegressor, verbosity, X, y) - samples, fitresult, cache, report, scores = ([], [], [], [], []) replacement = conf_model.replacement nsampling = conf_model.nsampling @@ -622,25 +611,26 @@ function MMI.fit(conf_model::JackknifePlusAbRegressor, verbosity, X, y) T = size(y, 1) # bootstrap size m = floor(Int, T * sample_size) - for _ = 1:nsampling - samplesᵢ = sample(1:T, m, replace = replacement) + for _ in 1:nsampling + samplesᵢ = sample(1:T, m; replace=replacement) yᵢ = y[samplesᵢ] Xᵢ = selectrows(X, samplesᵢ) - μ̂ᵢ, cacheᵢ, reportᵢ = - MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)...) + μ̂ᵢ, cacheᵢ, reportᵢ = MMI.fit( + conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)... + ) push!(samples, samplesᵢ) push!(fitresult, μ̂ᵢ) push!(cache, cacheᵢ) push!(report, reportᵢ) end - for t = 1:T + for t in 1:T index_samples = indexin([v for v in samples if !(t in v)], samples) selected_models = fitresult[index_samples] Xₜ = selectrows(X, t) yₜ = y[t] ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...), + MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...) ) for μ̂₋ₜ in selected_models ] ŷₜ = _aggregate(ŷ, aggregate) @@ -667,7 +657,7 @@ function MMI.predict(conf_model::JackknifePlusAbRegressor, fitresult, Xnew) # Get all bootstrapped predictions for each Xnew: ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...) ) for μ̂₋ᵢ in fitresult ] # Applying aggregation function on bootstrapped predictions across columns for each Xnew across rows: @@ -695,22 +685,15 @@ end function JackknifePlusAbMinMaxRegressor( model::Supervised; - coverage::AbstractFloat = 0.95, - heuristic::Function = f(y, ŷ) = abs(y - ŷ), - nsampling::Int = 30, - sample_size::AbstractFloat = 0.5, - replacement::Bool = true, - aggregate::Union{Symbol,String} = "mean", + coverage::AbstractFloat=0.95, + heuristic::Function=f(y, ŷ) = abs(y - ŷ), + nsampling::Int=30, + sample_size::AbstractFloat=0.5, + replacement::Bool=true, + aggregate::Union{Symbol,String}="mean", ) return JackknifePlusAbMinMaxRegressor( - model, - coverage, - nothing, - heuristic, - nsampling, - sample_size, - replacement, - aggregate, + model, coverage, nothing, heuristic, nsampling, sample_size, replacement, aggregate ) end @@ -726,7 +709,6 @@ S_i^{\text{J+MinMax}} = s(X_i, Y_i) = h(agg(\hat\mu_{B_{K(-i)}}(X_i)), Y_i), \ i where ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` denotes the aggregate predictions, typically mean or median, for each ``X_i`` (with ``K_{-i}`` the bootstraps not containing ``X_i``). In other words, B models are trained on boostrapped sampling, the fitted models are then used to create aggregated prediction of out-of-sample ``X_i``. The corresponding nonconformity score is then computed by applying a heuristic uncertainty measure ``h(\cdot)`` to the fitted value ``agg(\hat\mu_{B_{K(-i)}}(X_i))`` and the true value ``Y_i``. """ function MMI.fit(conf_model::JackknifePlusAbMinMaxRegressor, verbosity, X, y) - samples, fitresult, cache, report, scores = ([], [], [], [], []) replacement = conf_model.replacement nsampling = conf_model.nsampling @@ -735,25 +717,26 @@ function MMI.fit(conf_model::JackknifePlusAbMinMaxRegressor, verbosity, X, y) T = size(y, 1) # bootstrap size m = floor(Int, T * sample_size) - for _ = 1:nsampling - samplesᵢ = sample(1:T, m, replace = replacement) + for _ in 1:nsampling + samplesᵢ = sample(1:T, m; replace=replacement) yᵢ = y[samplesᵢ] Xᵢ = selectrows(X, samplesᵢ) - μ̂ᵢ, cacheᵢ, reportᵢ = - MMI.fit(conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)...) + μ̂ᵢ, cacheᵢ, reportᵢ = MMI.fit( + conf_model.model, 0, MMI.reformat(conf_model.model, Xᵢ, yᵢ)... + ) push!(samples, samplesᵢ) push!(fitresult, μ̂ᵢ) push!(cache, cacheᵢ) push!(report, reportᵢ) end - for t = 1:T + for t in 1:T index_samples = indexin([v for v in samples if !(t in v)], samples) selected_models = fitresult[index_samples] Xₜ = selectrows(X, t) yₜ = y[t] ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...), + MMI.predict(conf_model.model, μ̂₋ₜ, MMI.reformat(conf_model.model, Xₜ)...) ) for μ̂₋ₜ in selected_models ] ŷₜ = _aggregate(ŷ, aggregate) @@ -780,7 +763,7 @@ function MMI.predict(conf_model::JackknifePlusAbMinMaxRegressor, fitresult, Xnew # Get all bootstrapped predictions for each Xnew: ŷ = [ reformat_mlj_prediction( - MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...), + MMI.predict(conf_model.model, μ̂₋ᵢ, MMI.reformat(conf_model.model, Xnew)...) ) for μ̂₋ᵢ in fitresult ] ŷ = reduce(hcat, ŷ) diff --git a/src/conformal_models/utils.jl b/src/conformal_models/utils.jl index 9e2ab81..0514717 100644 --- a/src/conformal_models/utils.jl +++ b/src/conformal_models/utils.jl @@ -52,8 +52,7 @@ function set_size(ŷ) return _size end -function size_indicator(ŷ::AbstractVector; bins = 5, tol = 1e-10) - +function size_indicator(ŷ::AbstractVector; bins=5, tol=1e-10) _sizes = set_size.(ŷ) unique_sizes = unique(_sizes) @@ -63,18 +62,18 @@ function size_indicator(ŷ::AbstractVector; bins = 5, tol = 1e-10) idx = categorical(ones(length(_sizes))) else bin_caps = collect( - range(minimum(unique_sizes), maximum(unique_sizes), length = bins + 1), + range(minimum(unique_sizes), maximum(unique_sizes); length=bins + 1) )[2:end] idx = map(_sizes) do s # Check which is the largest bin cap that _size exceeds: ub = argmax(x -> s - x <= 0 ? s - x : -Inf, bin_caps) if ub == minimum(bin_caps) - ub = round(ub, digits = 2) - lb = round(minimum(_sizes), digits = 2) + ub = round(ub; digits=2) + lb = round(minimum(_sizes); digits=2) _idx = "|C| ∈ ($lb,$ub]" else - ub = round(ub, digits = 2) - lb = round(argmin(x -> s - x > 0 ? s - x : Inf, bin_caps), digits = 2) + ub = round(ub; digits=2) + lb = round(argmin(x -> s - x > 0 ? s - x : Inf, bin_caps); digits=2) _idx = "|C| ∈ ($lb,$ub]" end return _idx @@ -85,10 +84,10 @@ function size_indicator(ŷ::AbstractVector; bins = 5, tol = 1e-10) # Classification: if typeof(set_size(ŷ[1])) == Int - bin_caps = collect(1:2:(maximum(unique_sizes)+1)) + bin_caps = collect(1:2:(maximum(unique_sizes) + 1)) idx = map(_sizes) do s # Check which is the largest bin cap that _size exceeds: - ub = bin_caps[sum(s .> bin_caps)+1] + ub = bin_caps[sum(s .> bin_caps) + 1] if ub > maximum(_sizes) ub = ub - 1 _idx = "|C| ∈ [$ub]" @@ -102,5 +101,4 @@ function size_indicator(ŷ::AbstractVector; bins = 5, tol = 1e-10) end return idx - end diff --git a/src/evaluation/measures.jl b/src/evaluation/measures.jl index fa6498e..09ba9f4 100644 --- a/src/evaluation/measures.jl +++ b/src/evaluation/measures.jl @@ -19,7 +19,7 @@ function size_stratified_coverage(ŷ, y) # Setup: stratum_indicator = size_indicator(ŷ) |> x -> x.refs unique_stratums = sort(unique(stratum_indicator)) - unique_stratums = unique_stratums[unique_stratums.!=0] + unique_stratums = unique_stratums[unique_stratums .!= 0] _covs = [] if length(unique_stratums) == 1 && is_regression(ŷ) @@ -37,5 +37,4 @@ function size_stratified_coverage(ŷ, y) end return C̄ - end diff --git a/test/classification.jl b/test/classification.jl index d144934..32b28a7 100644 --- a/test/classification.jl +++ b/test/classification.jl @@ -9,23 +9,20 @@ data_specs = ( ) data_sets = Dict{String,Any}() for (k, v) in data_specs - X, y = MLJ.make_blobs(1000, v[1], centers = v[2]) + X, y = MLJ.make_blobs(1000, v[1]; centers=v[2]) X = MLJ.table(MLJ.matrix(X)) train, test = partition(eachindex(y), 0.8) _set = Dict(:data => (X, y), :split => (train, test), :specs => v) data_sets[k] = _set end - # Atomic and conformal models: models = tested_atomic_models[:classification] conformal_models = merge(values(available_models[:classification])...) # Test workflow: @testset "Classification" begin - for (model_name, import_call) in models - @testset "$(model_name)" begin # Import and instantiate atomic model: @@ -33,17 +30,15 @@ conformal_models = merge(values(available_models[:classification])...) model = Model() for _method in keys(conformal_models) - @testset "Method: $(_method)" begin # Instantiate conformal models: _cov = 0.85 - conf_model = conformal_model(model; method = _method, coverage = _cov) + conf_model = conformal_model(model; method=_method, coverage=_cov) conf_model = conformal_models[_method](model) @test isnothing(conf_model.scores) for (data_name, data_set) in data_sets - @testset "$(data_name)" begin # Unpack @@ -52,7 +47,7 @@ conformal_models = merge(values(available_models[:classification])...) # Fit/Predict: mach = machine(conf_model, X, y) - fit!(mach, rows = train) + fit!(mach; rows=train) @test !isnothing(conf_model.scores) predict(mach, selectrows(X, test)) @@ -60,17 +55,14 @@ conformal_models = merge(values(available_models[:classification])...) @test isplot(bar(mach.model, mach.fitresult, X)) @test isplot(areaplot(mach.model, mach.fitresult, X, y)) @test isplot( - areaplot(mach.model, mach.fitresult, X, y; input_var = 1), + areaplot(mach.model, mach.fitresult, X, y; input_var=1) ) @test isplot( - areaplot(mach.model, mach.fitresult, X, y; input_var = :x1), + areaplot(mach.model, mach.fitresult, X, y; input_var=:x1) ) if data_set[:specs][1] != 2 @test_throws AssertionError contourf( - mach.model, - mach.fitresult, - X, - y, + mach.model, mach.fitresult, X, y ) else @test isplot(contourf(mach.model, mach.fitresult, X, y)) @@ -80,12 +72,12 @@ conformal_models = merge(values(available_models[:classification])...) mach.fitresult, X, y; - zoom = -1, - plot_set_size = true, + zoom=-1, + plot_set_size=true, ), ) @test isplot( - contourf(mach.model, mach.fitresult, X, y; target = 1), + contourf(mach.model, mach.fitresult, X, y; target=1) ) end @@ -93,24 +85,16 @@ conformal_models = merge(values(available_models[:classification])...) # Evaluation takes some time, so only testing for one method. if _method == :simple_inductive && data_set[:specs][1] > 1 # Empirical coverage: - _eval = - evaluate!(mach; measure = emp_coverage, verbosity = 0) + _eval = evaluate!(mach; measure=emp_coverage, verbosity=0) Δ = _eval.measurement[1] - _cov # over-/under-coverage @test Δ >= -0.05 # we don't undercover too much # Size-stratified coverage: - _eval = evaluate!(mach; measure = ssc, verbosity = 0) + _eval = evaluate!(mach; measure=ssc, verbosity=0) end - end - end - end - end - end - end - end diff --git a/test/model_traits.jl b/test/model_traits.jl index 9206c81..6a04e51 100644 --- a/test/model_traits.jl +++ b/test/model_traits.jl @@ -4,12 +4,10 @@ Model = @load DecisionTreeClassifier pkg = DecisionTree model = Model() @testset "Model Traits" begin - @testset "Sampling Style" begin - conf_model = conformal_model(model; method = :naive) + conf_model = conformal_model(model; method=:naive) @test requires_data_splitting(conf_model) == false - conf_model = conformal_model(model; method = :simple_inductive) + conf_model = conformal_model(model; method=:simple_inductive) @test requires_data_splitting(conf_model) == true end - end diff --git a/test/regression.jl b/test/regression.jl index 80db55c..15f8f2c 100644 --- a/test/regression.jl +++ b/test/regression.jl @@ -2,8 +2,9 @@ using MLJ using Plots # Data: -data_specs = - ("Single Input - Single Output" => (1, 1), "Multiple Inputs - Single Output" => (5, 1)) +data_specs = ( + "Single Input - Single Output" => (1, 1), "Multiple Inputs - Single Output" => (5, 1) +) data_sets = Dict{String,Any}() for (k, v) in data_specs X, y = MLJ.make_regression(500, v[1]) @@ -19,9 +20,7 @@ conformal_models = merge(values(available_models[:regression])...) # Test workflow: @testset "Regression" begin - for (model_name, import_call) in models - @testset "$(model_name)" begin # Import and instantiate atomic model: @@ -29,17 +28,15 @@ conformal_models = merge(values(available_models[:regression])...) model = Model() for _method in keys(conformal_models) - @testset "Method: $(_method)" begin # Instantiate conformal models: _cov = 0.85 - conf_model = conformal_model(model; method = _method, coverage = _cov) + conf_model = conformal_model(model; method=_method, coverage=_cov) conf_model = conformal_models[_method](model) @test isnothing(conf_model.scores) for (data_name, data_set) in data_sets - @testset "$(data_name)" begin # Unpack: @@ -48,7 +45,7 @@ conformal_models = merge(values(available_models[:regression])...) # Fit/Predict: mach = machine(conf_model, X, y) - fit!(mach, rows = train) + fit!(mach; rows=train) @test !isnothing(conf_model.scores) predict(mach, selectrows(X, test)) @@ -60,13 +57,13 @@ conformal_models = merge(values(available_models[:regression])...) mach.fitresult, X, y; - input_var = 1, - xlims = (-1, 1), - ylims = (-1, 1), + input_var=1, + xlims=(-1, 1), + ylims=(-1, 1), ), ) @test isplot( - plot(mach.model, mach.fitresult, X, y; input_var = :x1), + plot(mach.model, mach.fitresult, X, y; input_var=:x1) ) @test isplot(bar(mach.model, mach.fitresult, X)) @@ -74,24 +71,16 @@ conformal_models = merge(values(available_models[:regression])...) # Evaluation takes some time, so only testing for one method. if _method == :simple_inductive # Empirical coverage: - _eval = - evaluate!(mach; measure = emp_coverage, verbosity = 0) + _eval = evaluate!(mach; measure=emp_coverage, verbosity=0) Δ = _eval.measurement[1] - _cov # over-/under-coverage @test Δ >= -0.05 # we don't undercover too much # Size-stratified coverage: - _eval = evaluate!(mach; measure = ssc, verbosity = 0) + _eval = evaluate!(mach; measure=ssc, verbosity=0) end - end - end - end - end - end - end - end diff --git a/test/runtests.jl b/test/runtests.jl index b01dab0..7a901ab 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,9 +11,7 @@ include("utils.jl") # Test suite: @testset "ConformalPrediction.jl" begin - include("classification.jl") include("regression.jl") include("model_traits.jl") - end