ENH: externalize MapieClassifier methods into several non-conformity scores

HISTORY.rst

@@ -5,6 +5,13 @@ History
 0.8.x (2024-xx-xx)
 ------------------
 
+* Extend `ConformityScore` to support regression (with `BaseRegressionScore`) and to support classification (with `BaseClassificationScore`)
+* Extend `EnsembleEstimator` to support regression (with `EnsembleRegressor`) and to support classification (with `EnsembleClassifier`)
+* Refactor `MapieClassifier` by separating the handling of the `MapieClassifier` estimator into a new class called `EnsembleClassifier`
+* Refactor `MapieClassifier` by separating the handling of the `MapieClassifier` conformity score into a new class called `BaseClassificationScore`
+* Add severals non-conformity scores for classification (`LAC`, `APS`, `RAPS`, `TopK`) based on `BaseClassificationScore`
+* Transfer the logic of classification methods into the non-conformity score classes (`LAC`, `APS`, `RAPS`, `TopK`)
+* Extend the classification strategy definition by supporting `method` and `conformity_score` attributes
 * Building unit tests for different `Subsample` and `BlockBooststrap` instances
 * Change the sign of C_k in the `Kolmogorov-Smirnov` test documentation
 * Building a training set with a fraction between 0 and 1 with `n_samples` attribute when using `split` method from `Subsample` class.

doc/api.rst

@@ -80,9 +80,14 @@ Conformity scores
    :toctree: generated/
    :template: class.rst
 
+   conformity_scores.BaseRegressionScore
    conformity_scores.AbsoluteConformityScore
    conformity_scores.GammaConformityScore
    conformity_scores.ResidualNormalisedScore
+   conformity_scores.BaseClassificationScore
+   conformity_scores.LAC
+   conformity_scores.APS
+   conformity_scores.TopK
 
 Resampling
 ==========

examples/classification/1-quickstart/plot_comp_methods_on_2d_dataset.py

@@ -170,7 +170,7 @@ def plot_scores(
 for i, method in enumerate(methods):
     conformity_scores = mapie[method].conformity_scores_
     n = mapie[method].n_samples_
-    quantiles = mapie[method].quantiles_
+    quantiles = mapie[method].conformity_score_function_.quantiles_
     plot_scores(alpha, conformity_scores, quantiles, method, axs[i])
 plt.show()
 
@@ -270,7 +270,8 @@ def plot_results(
 axs[0].set_xlabel("1 - alpha")
 axs[0].set_ylabel("Quantile")
 for method in methods:
-    axs[0].scatter(1 - alpha_, mapie[method].quantiles_, label=method)
+    quantiles = mapie[method].conformity_score_function_.quantiles_
+    axs[0].scatter(1 - alpha_, quantiles, label=method)
 axs[0].legend()
 for method in methods:
     axs[1].scatter(1 - alpha_, coverage[method], label=method)

examples/classification/4-tutorials/plot_crossconformal.py

@@ -134,10 +134,11 @@
 
 fig, axs = plt.subplots(1, len(mapies["lac"]), figsize=(20, 4))
 for i, (key, mapie) in enumerate(mapies["lac"].items()):
+    quantiles = mapie.conformity_score_function_.quantiles_[9]
     axs[i].set_xlabel("Conformity scores")
     axs[i].hist(mapie.conformity_scores_)
-    axs[i].axvline(mapie.quantiles_[9], ls="--", color="k")
-    axs[i].set_title(f"split={key}\nquantile={mapie.quantiles_[9]:.3f}")
+    axs[i].axvline(quantiles, ls="--", color="k")
+    axs[i].set_title(f"split={key}\nquantile={quantiles:.3f}")
 plt.suptitle(
     "Distribution of scores on each calibration fold for the "
     f"{methods[0]} method"

examples/classification/4-tutorials/plot_main-tutorial-binary-classification.py

@@ -188,7 +188,7 @@ def plot_scores(
 
 fig, axs = plt.subplots(1, 1, figsize=(10, 5))
 conformity_scores = mapie_clf.conformity_scores_
-quantiles = mapie_clf.quantiles_
+quantiles = mapie_clf.conformity_score_function_.quantiles_
 plot_scores(alpha, conformity_scores, quantiles, 'lac', axs)
 plt.show()
 
@@ -309,10 +309,11 @@ def plot_results(
 
 
 def plot_coverages_widths(alpha, coverage, width, method):
+    quantiles = mapie_clf.conformity_score_function_.quantiles_
     _, axs = plt.subplots(1, 3, figsize=(15, 5))
     axs[0].set_xlabel("1 - alpha")
     axs[0].set_ylabel("Quantile")
-    axs[0].scatter(1 - alpha, mapie_clf.quantiles_, label=method)
+    axs[0].scatter(1 - alpha, quantiles, label=method)
     axs[0].legend()
     axs[1].scatter(1 - alpha, coverage, label=method)
     axs[1].set_xlabel("1 - alpha")

examples/classification/4-tutorials/plot_main-tutorial-classification.py

@@ -148,7 +148,7 @@ def plot_scores(n, alphas, scores, quantiles):
 
 scores = mapie_score.conformity_scores_
 n = len(mapie_score.conformity_scores_)
-quantiles = mapie_score.quantiles_
+quantiles = mapie_score.conformity_score_function_.quantiles_
 plot_scores(n, alpha, scores, quantiles)
 
 ##############################################################################

examples/regression/2-advanced-analysis/plot_conformal_predictive_distribution.py

@@ -71,7 +71,7 @@ def get_cumulative_distribution_function(self, X):
         y_pred = self.predict(X)
         cs = self.conformity_scores_[~np.isnan(self.conformity_scores_)]
         res = self.conformity_score_function_.get_estimation_distribution(
-            X, y_pred.reshape((-1, 1)), cs
+            y_pred.reshape((-1, 1)), cs, X=X
         )
         return res
 

mapie/_machine_precision.py

@@ -1,5 +1,5 @@
 import numpy as np
 
-EPSILON = np.finfo(np.float64).eps
+EPSILON = np.float64(1e-8)
 
 __all__ = ["EPSILON"]