5 wrap models predict interval

Project.toml

@@ -5,13 +5,12 @@ version = "0.1.0"
 
 [deps]
 MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
-MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea"
-PkgTemplates = "14b8a8f1-9102-5b29-a752-f990bacb7fe1"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [compat]
 MLJ = "0.18"
+MLJModelInterface = "1"
 julia = "1.7"
 
 [extras]

docs/src/intro.qmd

@@ -64,11 +64,20 @@ fit!(mach, rows=train)
 calibrate!(conf_model, selectrows(X, calibration), y[calibration])
 ```
 
-Predictions can then be computed using the generic `predict` method. The code below produces predictions a random subset of test samples:
+Point predictions for the underlying machine learning model can be computed as always using the generic `predict` method. The code below produces predictions a random subset of test samples:
 
 ```{julia}
 #| output: true
-predict(conf_model, selectrows(X, rand(test,5)))
+Xtest = selectrows(X, rand(test,5))
+predict(mach, Xtest)
+```
+
+Conformal prediction regions can be computed using the `predict_region` method:
+
+```{julia}
+#| output: true
+coverage = .90
+predict_region(conf_model, Xtest, coverage)
 ```
 
 ## Contribute 🛠

src/ConformalModels/ConformalModels.jl

@@ -3,7 +3,6 @@ module ConformalModels
 using MLJ
 import MLJModelInterface as MMI
 import MLJModelInterface: predict, fit, save, restore
-import MLJBase 
 
 "An abstract base type for conformal models."
 abstract type ConformalModel <: MMI.Model end
@@ -44,6 +43,6 @@ const available_models = Dict(
 export available_models
 
 # Other general methods:
-export score, prediction_region
+export conformal_model, empirical_quantile, calibrate!, predict_region, score
 
 end

src/ConformalModels/conformal_models.jl

@@ -53,7 +53,6 @@ function MMI.fit(conf_model::InductiveConformalModel, verbosity, X, y)
     conf_model.fitresult = fitresult
     return (fitresult, cache, report)
 end
-export fit
 
 # Calibration
 """
@@ -66,8 +65,6 @@ function calibrate!(conf_model::InductiveConformalModel, Xcal, ycal)
     conf_model.scores = sort(ConformalModels.score(conf_model, Xcal, ycal), rev=true) # non-conformity scores
 end
 
-export calibrate!
-
 using Statistics
 """
     empirical_quantile(conf_model::ConformalModel, coverage::AbstractFloat=0.95)
@@ -83,7 +80,6 @@ function empirical_quantile(conf_model::ConformalModel, coverage::AbstractFloat=
     q̂ = Statistics.quantile(conf_model.scores, p̂)
     return q̂
 end
-export empirical_quantile
 
 # Prediction
 """
@@ -96,17 +92,6 @@ function MMI.predict(conf_model::ConformalModel, fitresult, Xnew)
     return yhat
 end
 
-# Conformal prediction through dispatch:
-"""
-    MMI.predict(conf_model::ConformalModel, Xnew, coverage::AbstractFloat=0.95)
-
-Computes the conformal prediction for any calibrated conformal model and new data `Xnew`. The default coverage ratio `(1-α)` is set to 95%.
-"""
-function MMI.predict(conf_model::ConformalModel, Xnew, coverage::AbstractFloat=0.95)
-    q̂ = empirical_quantile(conf_model, coverage)
-    return ConformalModels.prediction_region(conf_model, Xnew, q̂)
-end
-
 """
     score(conf_model::ConformalModel, Xcal, ycal)
 
@@ -117,10 +102,10 @@ function score(conf_model::ConformalModel, Xcal, ycal)
 end
 
 """
-    prediction_region(conf_model::ConformalModel, Xnew, q̂::Real)
+    predict_region(conf_model::ConformalModel, Xnew, coverage::AbstractFloat=0.95)
 
 Generic method for generating prediction regions from a calibrated conformal model for a given quantile.
 """
-function prediction_region(conf_model::ConformalModel, Xnew, q̂::Real)
+function predict_region(conf_model::ConformalModel, Xnew, coverage::AbstractFloat=0.95)
     # pass
 end

src/ConformalModels/inductive_classification.jl

@@ -1,6 +1,20 @@
 "A base type for Inductive Conformal Classifiers."
 abstract type InductiveConformalClassifier <: InductiveConformalModel end
 
+"""
+    predict_region(conf_model::InductiveConformalClassifier, Xnew, coverage::AbstractFloat=0.95)
+
+Generic method to compute prediction region for given quantile `q̂` for Inductive Conformal Classifiers. 
+"""
+function predict_region(conf_model::InductiveConformalClassifier, Xnew, coverage::AbstractFloat=0.95)
+    q̂ = empirical_quantile(conf_model, coverage)
+    p̂ = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
+    L = p̂.decoder.classes
+    ŷnew = pdf(p̂, L)
+    ŷnew = map(x -> collect(key => 1-val <= q̂ ? val : missing for (key,val) in zip(L,x)),eachrow(ŷnew))
+    return ŷnew 
+end
+
 # Simple
 "The `SimpleInductiveClassifier` is the simplest approach to Inductive Conformal Classification. Contrary to the [`NaiveClassifier`](@ref) it computes nonconformity scores using a designated calibration dataset."
 mutable struct SimpleInductiveClassifier{Model <: Supervised} <: InductiveConformalClassifier
@@ -19,11 +33,3 @@ function score(conf_model::SimpleInductiveClassifier, Xcal, ycal)
     return @.(1.0 - ŷ)
 end
 
-function prediction_region(conf_model::SimpleInductiveClassifier, Xnew, q̂::Real)
-    p̂ = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
-    L = p̂.decoder.classes
-    ŷnew = pdf(p̂, L)
-    ŷnew = map(x -> collect(key => 1-val <= q̂::Real ? val : missing for (key,val) in zip(L,x)),eachrow(ŷnew))
-    return ŷnew 
-end
-

src/ConformalModels/inductive_regression.jl

@@ -1,6 +1,18 @@
 "A base type for Inductive Conformal Regressors."
 abstract type InductiveConformalRegressor <: InductiveConformalModel end
 
+"""
+    predict_region(conf_model::InductiveConformalRegressor, Xnew, coverage::AbstractFloat=0.95)
+
+Generic method to compute prediction region for given quantile `q̂` for Inductive Conformal Regressors. 
+"""
+function predict_region(conf_model::InductiveConformalRegressor, Xnew, coverage::AbstractFloat=0.95)
+    q̂ = empirical_quantile(conf_model, coverage)
+    ŷnew = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
+    ŷnew = map(x -> ["lower" => x .- q̂, "upper" => x .+ q̂],eachrow(ŷnew))
+    return ŷnew 
+end
+
 "The `SimpleInductiveRegressor` is the simplest approach to Inductive Conformal Regression. Contrary to the [`NaiveRegressor`](@ref) it computes nonconformity scores using a designated calibration dataset."
 mutable struct SimpleInductiveRegressor{Model <: Supervised} <: InductiveConformalRegressor
     model::Model
@@ -17,8 +29,3 @@ function score(conf_model::SimpleInductiveRegressor, Xcal, ycal)
     return @.(abs(ŷ - ycal))
 end
 
-function prediction_region(conf_model::SimpleInductiveRegressor, Xnew, q̂::Real)
-    ŷnew = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
-    ŷnew = map(x -> ["lower" => x .- q̂, "upper" => x .+ q̂],eachrow(ŷnew))
-    return ŷnew 
-end

src/ConformalModels/transductive_classification.jl

@@ -1,6 +1,20 @@
 "A base type for Transductive Conformal Classifiers."
 abstract type TransductiveConformalClassifier <: TransductiveConformalModel end
 
+"""
+    predict_region(conf_model::TransductiveConformalClassifier, Xnew, coverage::AbstractFloat=0.95)
+
+Generic method to compute prediction region for given quantile `q̂` for Transductive Conformal Classifiers. 
+"""
+function predict_region(conf_model::TransductiveConformalClassifier, Xnew, coverage::AbstractFloat=0.95)
+    q̂ = empirical_quantile(conf_model, coverage)
+    p̂ = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
+    L = p̂.decoder.classes
+    ŷnew = pdf(p̂, L)
+    ŷnew = map(x -> collect(key => 1-val <= q̂::Real ? val : missing for (key,val) in zip(L,x)),eachrow(ŷnew))
+    return ŷnew 
+end
+
 # Simple
 "The `NaiveClassifier` is the simplest approach to Inductive Conformal Classification. Contrary to the [`NaiveClassifier`](@ref) it computes nonconformity scores using a designated trainibration dataset."
 mutable struct NaiveClassifier{Model <: Supervised} <: TransductiveConformalClassifier
@@ -17,12 +31,4 @@ end
 function score(conf_model::NaiveClassifier, Xtrain, ytrain)
     ŷ = pdf.(MMI.predict(conf_model.model, conf_model.fitresult, Xtrain),ytrain)
     return @.(1.0 - ŷ)
-end
-
-function prediction_region(conf_model::NaiveClassifier, Xnew, q̂::Real)
-    p̂ = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
-    L = p̂.decoder.classes
-    ŷnew = pdf(p̂, L)
-    ŷnew = map(x -> collect(key => 1-val <= q̂::Real ? val : missing for (key,val) in zip(L,x)),eachrow(ŷnew))
-    return ŷnew 
 end

src/ConformalModels/transductive_regression.jl

@@ -3,6 +3,18 @@ using MLJ
 "A base type for Transductive Conformal Regressors."
 abstract type TransductiveConformalRegressor <: TransductiveConformalModel end
 
+"""
+    predict_region(conf_model::TransductiveConformalRegressor, Xnew, coverage::AbstractFloat=0.95)
+
+Generic method to compute prediction region for given quantile `q̂` for Transductive Conformal Regressors. 
+"""
+function predict_region(conf_model::TransductiveConformalRegressor, Xnew, coverage::AbstractFloat=0.95)
+    q̂ = empirical_quantile(conf_model, coverage)
+    ŷnew = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
+    ŷnew = map(x -> ["lower" => x .- q̂, "upper" => x .+ q̂],eachrow(ŷnew))
+    return ŷnew 
+end
+
 # Naive
 "The `NaiveRegressor` for conformal prediction is the simplest approach to conformal regression. It computes nonconformity scores by simply using the training data."
 mutable struct NaiveRegressor{Model <: Supervised} <: TransductiveConformalRegressor
@@ -20,12 +32,6 @@ function score(conf_model::NaiveRegressor, Xtrain, ytrain)
     return @.(abs(ŷ - ytrain))
 end
 
-function prediction_region(conf_model::NaiveRegressor, Xnew, q̂::Real)
-    ŷnew = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
-    ŷnew = map(x -> ["lower" => x .- q̂, "upper" => x .+ q̂],eachrow(ŷnew))
-    return ŷnew 
-end
-
 # Jackknife
 "The `Jackknife` ..."
 mutable struct JackknifeRegressor{Model <: Supervised} <: TransductiveConformalRegressor
@@ -53,11 +59,7 @@ function score(conf_model::JackknifeRegressor, Xtrain, ytrain)
     return scores
 end
 
-function prediction_region(conf_model::JackknifeRegressor, Xnew, q̂::Real)
-    ŷnew = MMI.predict(conf_model.model, conf_model.fitresult, Xnew)
-    ŷnew = map(x -> ["lower" => x .- q̂, "upper" => x .+ q̂],eachrow(ŷnew))
-    return ŷnew 
-end
+
 
 
 

src/ConformalPrediction.jl

@@ -3,7 +3,7 @@ module ConformalPrediction
 # conformal models
 include("ConformalModels/ConformalModels.jl")
 using .ConformalModels
-export conformal_model, fit, calibrate!
+export conformal_model, empirical_quantile, calibrate!, predict_region, score
 export NaiveRegressor, SimpleInductiveRegressor, JackknifeRegressor
 export NaiveClassifier, SimpleInductiveClassifier
 export available_models

test/classification.jl

@@ -31,8 +31,10 @@ available_models = ConformalPrediction.ConformalModels.available_models[:classif
                 fit!(_mach, rows=train)
                 calibrate!(conf_model, selectrows(X, calibration), y[calibration])
 
+                # Prediction:
                 @test !isnothing(conf_model.scores)
-                predict(conf_model, selectrows(X, test))
+                predict(_mach, selectrows(X, test))                 # point predictions
+                predict_region(conf_model, selectrows(X, test))     # prediction region
             end
         end
     end
@@ -53,8 +55,10 @@ available_models = ConformalPrediction.ConformalModels.available_models[:classif
                 _mach = machine(conf_model, X, y)
                 fit!(_mach, rows=train)
 
+                # Prediction:
                 @test !isnothing(conf_model.scores)
-                predict(conf_model, selectrows(X, test))
+                predict(_mach, selectrows(X, test))                 # point predictions
+                predict_region(conf_model, selectrows(X, test))     # prediction region
             end
         end
 

test/regression.jl

@@ -32,8 +32,10 @@ available_models = ConformalPrediction.ConformalModels.available_models[:regress
                 fit!(_mach, rows=train)
                 calibrate!(conf_model, selectrows(X, calibration), y[calibration])
 
+                # Prediction:
                 @test !isnothing(conf_model.scores)
-                predict(conf_model, selectrows(X, test))
+                predict(_mach, selectrows(X, test))                 # point predictions
+                predict_region(conf_model, selectrows(X, test))     # prediction region
             end
         end
     end
@@ -54,8 +56,10 @@ available_models = ConformalPrediction.ConformalModels.available_models[:regress
                 _mach = machine(conf_model, X, y)
                 fit!(_mach, rows=train)
 
-                @test !isnothing(conf_model.scores)
-                predict(conf_model, selectrows(X, test))
+                 # Prediction:
+                 @test !isnothing(conf_model.scores)
+                 predict(_mach, selectrows(X, test))                 # point predictions
+                 predict_region(conf_model, selectrows(X, test))     # prediction region
             end
         end