diff --git a/Project.toml b/Project.toml index 9754213..244dfeb 100755 --- a/Project.toml +++ b/Project.toml @@ -22,6 +22,7 @@ ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +TaijaBase = "10284c91-9f28-4c9a-abbf-ee43576dfff6" [compat] Aqua = "0.8" diff --git a/src/ConformalPrediction.jl b/src/ConformalPrediction.jl index eafc829..9164852 100755 --- a/src/ConformalPrediction.jl +++ b/src/ConformalPrediction.jl @@ -1,5 +1,7 @@ module ConformalPrediction +using TaijaBase + # Conformal Models: include("conformal_models/conformal_models.jl") export ConformalModel diff --git a/src/conformal_models/conformal_models.jl b/src/conformal_models/conformal_models.jl index 32bba68..7ad9913 100755 --- a/src/conformal_models/conformal_models.jl +++ b/src/conformal_models/conformal_models.jl @@ -51,13 +51,14 @@ function conformal_model( return conf_model end +# Inductive Models: +include("inductive_bayes_regression.jl") +include("inductive/inductive_models.jl") + # Regression Models: -include("inductive_regression.jl") include("transductive_regression.jl") -include("inductive_bayes_regression.jl") + # Classification Models -include("inductive_classification.jl") -#include("inductive_bayes_classification.jl") include("transductive_classification.jl") # Training: @@ -65,14 +66,6 @@ include("ConformalTraining/ConformalTraining.jl") using .ConformalTraining # Type unions: -const InductiveModel = Union{ - SimpleInductiveRegressor, - SimpleInductiveClassifier, - AdaptiveInductiveClassifier, - ConformalQuantileRegressor, - BayesRegressor, -} - const TransductiveModel = Union{ NaiveRegressor, JackknifeRegressor, diff --git a/src/conformal_models/inductive_classification.jl b/src/conformal_models/inductive/classification.jl similarity index 78% rename from src/conformal_models/inductive_classification.jl rename to src/conformal_models/inductive/classification.jl index fd7fda3..5b70c59 100755 --- a/src/conformal_models/inductive_classification.jl +++ b/src/conformal_models/inductive/classification.jl @@ -14,6 +14,7 @@ mutable struct SimpleInductiveClassifier{Model<:Supervised} <: ConformalProbabil coverage::AbstractFloat scores::Union{Nothing,Dict{Any,Any}} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} train_ratio::AbstractFloat end @@ -21,15 +22,24 @@ function SimpleInductiveClassifier( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=minus_softmax, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, train_ratio::AbstractFloat=0.5, ) - return SimpleInductiveClassifier(model, coverage, nothing, heuristic, train_ratio) + return SimpleInductiveClassifier( + model, coverage, nothing, heuristic, parallelizer, train_ratio + ) end -""" +@doc raw""" score(conf_model::SimpleInductiveClassifier, ::Type{<:Supervised}, fitresult, X, y::Union{Nothing,AbstractArray}=nothing) -Score method for the [`SimpleInductiveClassifier`](@ref) dispatched for any `<:Supervised` model. +Score method for the [`SimpleInductiveClassifier`](@ref) dispatched for any `<:Supervised` model. For the [`SimpleInductiveClassifier`](@ref) nonconformity scores are computed as follows: + +`` +S_i^{\text{CAL}} = s(X_i, Y_i) = h(\hat\mu(X_i), Y_i), \ i \in \mathcal{D}_{\text{calibration}} +`` + +A typical choice for the heuristic function is ``h(\hat\mu(X_i), Y_i)=1-\hat\mu(X_i)_{Y_i}`` where ``\hat\mu(X_i)_{Y_i}`` denotes the softmax output of the true class and ``\hat\mu`` denotes the model fitted on training data ``\mathcal{D}_{\text{train}}``. The simple approach only takes the softmax probability of the true label into account. """ function score( conf_model::SimpleInductiveClassifier, atomic::Supervised, fitresult, X, y=nothing @@ -46,34 +56,6 @@ function score( end end -@doc raw""" - MMI.fit(conf_model::SimpleInductiveClassifier, verbosity, X, y) - -For the [`SimpleInductiveClassifier`](@ref) nonconformity scores are computed as follows: - -`` -S_i^{\text{CAL}} = s(X_i, Y_i) = h(\hat\mu(X_i), Y_i), \ i \in \mathcal{D}_{\text{calibration}} -`` - -A typical choice for the heuristic function is ``h(\hat\mu(X_i), Y_i)=1-\hat\mu(X_i)_{Y_i}`` where ``\hat\mu(X_i)_{Y_i}`` denotes the softmax output of the true class and ``\hat\mu`` denotes the model fitted on training data ``\mathcal{D}_{\text{train}}``. The simple approach only takes the softmax probability of the true label into account. -""" -function MMI.fit(conf_model::SimpleInductiveClassifier, verbosity, X, y) - - # Data Splitting: - Xtrain, ytrain, Xcal, ycal = split_data(conf_model, X, y) - - # Training: - fitresult, cache, report = MMI.fit( - conf_model.model, verbosity, MMI.reformat(conf_model.model, Xtrain, ytrain)... - ) - - # Nonconformity Scores: - cal_scores, scores = score(conf_model, fitresult, Xcal, ycal) - conf_model.scores = Dict(:calibration => cal_scores, :all => scores) - - return (fitresult, cache, report) -end - @doc raw""" MMI.predict(conf_model::SimpleInductiveClassifier, fitresult, Xnew) @@ -112,6 +94,7 @@ mutable struct AdaptiveInductiveClassifier{Model<:Supervised} <: ConformalProbab coverage::AbstractFloat scores::Union{Nothing,Dict{Any,Any}} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} train_ratio::AbstractFloat end @@ -119,35 +102,12 @@ function AdaptiveInductiveClassifier( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=minus_softmax, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, train_ratio::AbstractFloat=0.5, ) - return AdaptiveInductiveClassifier(model, coverage, nothing, heuristic, train_ratio) -end - -@doc raw""" - MMI.fit(conf_model::AdaptiveInductiveClassifier, verbosity, X, y) - -For the [`AdaptiveInductiveClassifier`](@ref) nonconformity scores are computed by cumulatively summing the ranked scores of each label in descending order until reaching the true label ``Y_i``: - -`` -S_i^{\text{CAL}} = s(X_i,Y_i) = \sum_{j=1}^k \hat\mu(X_i)_{\pi_j} \ \text{where } \ Y_i=\pi_k, i \in \mathcal{D}_{\text{calibration}} -`` -""" -function MMI.fit(conf_model::AdaptiveInductiveClassifier, verbosity, X, y) - - # Data Splitting: - Xtrain, ytrain, Xcal, ycal = split_data(conf_model, X, y) - - # Training: - fitresult, cache, report = MMI.fit( - conf_model.model, verbosity, MMI.reformat(conf_model.model, Xtrain, ytrain)... + return AdaptiveInductiveClassifier( + model, coverage, nothing, heuristic, parallelizer, train_ratio ) - - # Nonconformity Scores: - cal_scores, scores = score(conf_model, fitresult, Xcal, ycal) - conf_model.scores = Dict(:calibration => cal_scores, :all => scores) - - return (fitresult, cache, report) end """ diff --git a/src/conformal_models/inductive/inductive_models.jl b/src/conformal_models/inductive/inductive_models.jl new file mode 100644 index 0000000..412dd7b --- /dev/null +++ b/src/conformal_models/inductive/inductive_models.jl @@ -0,0 +1,67 @@ +# Type unions: +include("classification.jl") +include("regression.jl") + +const InductiveModel = Union{ + SimpleInductiveRegressor, + SimpleInductiveClassifier, + AdaptiveInductiveClassifier, + ConformalQuantileRegressor, + BayesRegressor, +} + +""" + split_data(conf_model::InductiveModel, indices::Base.OneTo{Int}) + +Splits the data into a proper training and calibration set for inductive models. +""" +function split_data(conf_model::InductiveModel, X, y) + train, calibration = partition(eachindex(y), conf_model.train_ratio) + Xtrain = selectrows(X, train) + ytrain = y[train] + Xcal = selectrows(X, calibration) + ycal = y[calibration] + + return Xtrain, ytrain, Xcal, ycal +end + +""" + score(conf_model::InductiveModel, fitresult, X, y=nothing) + +Generic score method for the [`InductiveModel`](@ref). It computes nonconformity scores using the heuristic function `h` and the softmax probabilities of the true class. Method is dispatched for different Conformal Probabilistic Sets and atomic models. +""" +function score(conf_model::InductiveModel, fitresult, X, y=nothing) + return score(conf_model, conf_model.model, fitresult, X, y) +end + +""" + fit_atomic(conf_model::InductiveModel, verbosity, X, y) + +Fits the atomic model for the [`InductiveModel`](@ref). In the case of inductive models, the atomic model is fit once on the proper training data. +""" +function fit_atomic(conf_model::InductiveModel, verbosity, X, y) + fitresult, cache, report = MMI.fit( + conf_model.model, verbosity, MMI.reformat(conf_model.model, X, y)... + ) + return fitresult, cache, report +end + +@doc raw""" + MMI.fit(conf_model::InductiveModel, verbosity, X, y) + +Fits the [`InductiveModel`](@ref) model. +""" +function MMI.fit(conf_model::InductiveModel, verbosity, X, y) + + # Data Splitting: + Xtrain, ytrain, Xcal, ycal = split_data(conf_model, X, y) + + # Training: + fitresult, cache, report = fit_atomic(conf_model, verbosity, Xtrain, ytrain) + + # Nonconformity Scores: + cal_scores, scores = score(conf_model, fitresult, Xcal, ycal) + conf_model.scores = Dict(:calibration => cal_scores, :all => scores) + + return (fitresult, cache, report) +end diff --git a/src/conformal_models/inductive_regression.jl b/src/conformal_models/inductive/regression.jl similarity index 81% rename from src/conformal_models/inductive_regression.jl rename to src/conformal_models/inductive/regression.jl index bfbc758..008f3c7 100755 --- a/src/conformal_models/inductive_regression.jl +++ b/src/conformal_models/inductive/regression.jl @@ -4,8 +4,9 @@ using MLJLinearModels: MLJLinearModels mutable struct SimpleInductiveRegressor{Model<:Supervised} <: ConformalInterval model::Model coverage::AbstractFloat - scores::Union{Nothing,AbstractArray} + scores::Union{Nothing,Dict{Any,Any}} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} train_ratio::AbstractFloat end @@ -13,13 +14,16 @@ function SimpleInductiveRegressor( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, train_ratio::AbstractFloat=0.5, ) - return SimpleInductiveRegressor(model, coverage, nothing, heuristic, train_ratio) + return SimpleInductiveRegressor( + model, coverage, nothing, heuristic, parallelizer, train_ratio + ) end @doc raw""" - MMI.fit(conf_model::SimpleInductiveRegressor, verbosity, X, y) + score(conf_model::SimpleInductiveRegressor, atomic::Supervised, fitresult, X, y=nothing) For the [`SimpleInductiveRegressor`](@ref) nonconformity scores are computed as follows: @@ -29,22 +33,16 @@ S_i^{\text{CAL}} = s(X_i, Y_i) = h(\hat\mu(X_i), Y_i), \ i \in \mathcal{D}_{\tex A typical choice for the heuristic function is ``h(\hat\mu(X_i),Y_i)=|Y_i-\hat\mu(X_i)|`` where ``\hat\mu`` denotes the model fitted on training data ``\mathcal{D}_{\text{train}}``. """ -function MMI.fit(conf_model::SimpleInductiveRegressor, verbosity, X, y) - - # Data Splitting: - Xtrain, ytrain, Xcal, ycal = split_data(conf_model, X, y) - # Training: - fitresult, cache, report = MMI.fit( - conf_model.model, verbosity, MMI.reformat(conf_model.model, Xtrain, ytrain)... - ) - - # Nonconformity Scores: - ŷ = reformat_mlj_prediction( - MMI.predict(conf_model.model, fitresult, MMI.reformat(conf_model.model, Xcal)...) - ) - conf_model.scores = @.(conf_model.heuristic(ycal, ŷ)) - - return (fitresult, cache, report) +function score( + conf_model::SimpleInductiveRegressor, atomic::Supervised, fitresult, X, y=nothing +) + ŷ = reformat_mlj_prediction(MMI.predict(atomic, fitresult, MMI.reformat(atomic, X)...)) + scores = @.(conf_model.heuristic(y, ŷ)) + if isnothing(y) + return scores + else + return scores, scores + end end # Prediction @@ -79,6 +77,7 @@ mutable struct ConformalQuantileRegressor{Model<:QuantileModel} <: ConformalInte coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} train_ratio::AbstractFloat end @@ -88,11 +87,19 @@ function ConformalQuantileRegressor( heuristic::Function=function f(y, ŷ_lb, ŷ_ub) return reduce((x, y) -> max.(x, y), [ŷ_lb - y, y - ŷ_ub]) end, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, train_ratio::AbstractFloat=0.5, ) - return ConformalQuantileRegressor(model, coverage, nothing, heuristic, train_ratio) + return ConformalQuantileRegressor( + model, coverage, nothing, heuristic, parallelizer, train_ratio + ) end +# function fit_atomic(conf_model::ConformalQuantileRegressor, verbosity, X, y) +# fitresult, cache, report = MMI.fit(conf_model.model, verbosity, MMI.reformat(conf_model.model, X, y)...) +# return fitresult, cache, report +# end + @doc raw""" MMI.fit(conf_model::ConformalQuantileRegressor, verbosity, X, y) @@ -109,13 +116,7 @@ A typical choice for the heuristic function is ``h(\hat\mu_{\alpha_{lo}}(X_i), \ function MMI.fit(conf_model::ConformalQuantileRegressor, verbosity, X, y) # Data Splitting: - train, calibration = partition(eachindex(y), conf_model.train_ratio) - Xtrain = selectrows(X, train) - ytrain = y[train] - Xtrain, ytrain = MMI.reformat(conf_model.model, Xtrain, ytrain) - Xcal = selectrows(X, calibration) - ycal = y[calibration] - Xcal, ycal = MMI.reformat(conf_model.model, Xcal, ycal) + Xtrain, ytrain, Xcal, ycal = split_data(conf_model, X, y) # Training: fitresult, cache, report, y_pred = ([], [], [], []) diff --git a/src/conformal_models/inductive_bayes_regression.jl b/src/conformal_models/inductive_bayes_regression.jl index bab9202..5cb02de 100644 --- a/src/conformal_models/inductive_bayes_regression.jl +++ b/src/conformal_models/inductive_bayes_regression.jl @@ -1,5 +1,6 @@ using LaplaceRedux: LaplaceRegression + @doc raw""" The `BayesRegressor` is the simplest approach to Inductive Conformalized Bayes. As explained in https://arxiv.org/abs/2107.07511, the conformal score is defined as the opposite of the probability of observing y given x : `` s= -P(Y|X) ``. Once the treshold ``\hat{q}`` is chosen, The credible interval is then diff --git a/src/conformal_models/transductive_classification.jl b/src/conformal_models/transductive_classification.jl index d13da08..e489670 100755 --- a/src/conformal_models/transductive_classification.jl +++ b/src/conformal_models/transductive_classification.jl @@ -5,12 +5,16 @@ mutable struct NaiveClassifier{Model<:Supervised} <: ConformalProbabilisticSet coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} end function NaiveClassifier( - model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=minus_softmax + model::Supervised; + coverage::AbstractFloat=0.95, + heuristic::Function=minus_softmax, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, ) - return NaiveClassifier(model, coverage, nothing, heuristic) + return NaiveClassifier(model, coverage, nothing, heuristic, parallelizer) end @doc raw""" diff --git a/src/conformal_models/transductive_regression.jl b/src/conformal_models/transductive_regression.jl index 6598531..a07e762 100755 --- a/src/conformal_models/transductive_regression.jl +++ b/src/conformal_models/transductive_regression.jl @@ -10,12 +10,16 @@ mutable struct NaiveRegressor{Model<:Supervised} <: ConformalInterval coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} end function NaiveRegressor( - model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error + model::Supervised; + coverage::AbstractFloat=0.95, + heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, ) - return NaiveRegressor(model, coverage, nothing, heuristic) + return NaiveRegressor(model, coverage, nothing, heuristic, parallelizer) end @doc raw""" @@ -79,12 +83,16 @@ mutable struct JackknifeRegressor{Model<:Supervised} <: ConformalInterval coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} end function JackknifeRegressor( - model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error + model::Supervised; + coverage::AbstractFloat=0.95, + heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, ) - return JackknifeRegressor(model, coverage, nothing, heuristic) + return JackknifeRegressor(model, coverage, nothing, heuristic, parallelizer) end @doc raw""" @@ -160,12 +168,16 @@ mutable struct JackknifePlusRegressor{Model<:Supervised} <: ConformalInterval coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} end function JackknifePlusRegressor( - model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error + model::Supervised; + coverage::AbstractFloat=0.95, + heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, ) - return JackknifePlusRegressor(model, coverage, nothing, heuristic) + return JackknifePlusRegressor(model, coverage, nothing, heuristic, parallelizer) end @doc raw""" @@ -249,12 +261,16 @@ mutable struct JackknifeMinMaxRegressor{Model<:Supervised} <: ConformalInterval coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} end function JackknifeMinMaxRegressor( - model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error + model::Supervised; + coverage::AbstractFloat=0.95, + heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, ) - return JackknifeMinMaxRegressor(model, coverage, nothing, heuristic) + return JackknifeMinMaxRegressor(model, coverage, nothing, heuristic, parallelizer) end @doc raw""" @@ -337,6 +353,7 @@ mutable struct CVPlusRegressor{Model<:Supervised} <: ConformalInterval coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} cv::MLJBase.CV end @@ -344,9 +361,10 @@ function CVPlusRegressor( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, cv::MLJBase.CV=MLJBase.CV(), ) - return CVPlusRegressor(model, coverage, nothing, heuristic, cv) + return CVPlusRegressor(model, coverage, nothing, heuristic, parallelizer, cv) end @doc raw""" @@ -442,6 +460,7 @@ mutable struct CVMinMaxRegressor{Model<:Supervised} <: ConformalInterval coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} cv::MLJBase.CV end @@ -449,9 +468,10 @@ function CVMinMaxRegressor( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, cv::MLJBase.CV=MLJBase.CV(), ) - return CVMinMaxRegressor(model, coverage, nothing, heuristic, cv) + return CVMinMaxRegressor(model, coverage, nothing, heuristic, parallelizer, cv) end @doc raw""" @@ -567,6 +587,7 @@ mutable struct JackknifePlusAbRegressor{Model<:Supervised} <: ConformalInterval coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} nsampling::Int sample_size::AbstractFloat replacement::Bool @@ -577,13 +598,22 @@ function JackknifePlusAbRegressor( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, nsampling::Int=30, sample_size::AbstractFloat=0.5, replacement::Bool=true, aggregate::Union{Symbol,String}="mean", ) return JackknifePlusAbRegressor( - model, coverage, nothing, heuristic, nsampling, sample_size, replacement, aggregate + model, + coverage, + nothing, + heuristic, + parallelizer, + nsampling, + sample_size, + replacement, + aggregate, ) end @@ -673,6 +703,7 @@ mutable struct JackknifePlusAbMinMaxRegressor{Model<:Supervised} <: ConformalInt coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} nsampling::Int sample_size::AbstractFloat replacement::Bool @@ -683,13 +714,22 @@ function JackknifePlusAbMinMaxRegressor( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, nsampling::Int=30, sample_size::AbstractFloat=0.5, replacement::Bool=true, aggregate::Union{Symbol,String}="mean", ) return JackknifePlusAbMinMaxRegressor( - model, coverage, nothing, heuristic, nsampling, sample_size, replacement, aggregate + model, + coverage, + nothing, + heuristic, + parallelizer, + nsampling, + sample_size, + replacement, + aggregate, ) end @@ -777,6 +817,7 @@ mutable struct TimeSeriesRegressorEnsembleBatch{Model<:Supervised} <: ConformalI coverage::AbstractFloat scores::Union{Nothing,AbstractArray} heuristic::Function + parallelizer::Union{Nothing,AbstractParallelizer} nsampling::Int sample_size::AbstractFloat aggregate::Union{Symbol,String} @@ -786,12 +827,13 @@ function TimeSeriesRegressorEnsembleBatch( model::Supervised; coverage::AbstractFloat=0.95, heuristic::Function=absolute_error, + parallelizer::Union{Nothing,AbstractParallelizer}=nothing, nsampling::Int=50, sample_size::AbstractFloat=0.3, aggregate::Union{Symbol,String}="mean", ) return TimeSeriesRegressorEnsembleBatch( - model, coverage, nothing, heuristic, nsampling, sample_size, aggregate + model, coverage, nothing, heuristic, parallelizer, nsampling, sample_size, aggregate ) end