Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FIX use balanced accuracy from scikit-learn #128

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 18 additions & 12 deletions rampwf/score_types/balanced_accuracy.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,21 @@
"""Balanced accuracy.
from .classifier_base import ClassifierBaseScoreType
from sklearn.metrics import recall_score
from sklearn.metrics.classification import _check_targets

From https://github.com/ch-imad/AutoMl_Challenge/blob/2353ec0/Starting_kit/scoring_program/libscores.py#L187 # noqa

See the thread at
https://github.com/rhiever/tpot/issues/108#issuecomment-317067760
about the different definitions.
"""
from .classifier_base import ClassifierBaseScoreType
from .macro_averaged_recall import MacroAveragedRecall
def _balanced_accuracy_score(y_true, y_pred, sample_weight=None):
"""FIXME: port implementation of balanced accuracy from scikit-learn 0.20.
"""
y_type, y_true, y_pred = _check_targets(y_true, y_pred)

if y_type != 'binary':
raise ValueError('Balanced accuracy is only meaningful '
'for binary classification problems.')
# simply wrap the ``recall_score`` function
return recall_score(y_true, y_pred,
pos_label=None,
average='macro',
sample_weight=sample_weight)


class BalancedAccuracy(ClassifierBaseScoreType):
Expand All @@ -20,8 +28,6 @@ def __init__(self, name='balanced_accuracy', precision=2):
self.precision = precision

def __call__(self, y_true_label_index, y_pred_label_index):
mac = MacroAveragedRecall()
tpr = mac(y_true_label_index, y_pred_label_index)
base_tpr = 1. / len(self.label_names)
score = (tpr - base_tpr) / (1 - base_tpr)
score = _balanced_accuracy_score(y_true_label_index,
y_pred_label_index)
return score