diff --git a/arekit/contrib/experiment_rusentrel/evaluators/__init__.py b/arekit/contrib/experiment_rusentrel/evaluators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/arekit/contrib/experiment_rusentrel/evaluators/three_class_opinion.py b/arekit/contrib/experiment_rusentrel/evaluators/three_class_opinion.py new file mode 100644 index 00000000..7cfe0dea --- /dev/null +++ b/arekit/contrib/experiment_rusentrel/evaluators/three_class_opinion.py @@ -0,0 +1,55 @@ +from arekit.common.evaluation.comparators.opinions import OpinionBasedComparator +from arekit.common.evaluation.evaluators.base import BaseEvaluator +from arekit.common.evaluation.evaluators.modes import EvaluationModes +from arekit.common.evaluation.evaluators.utils import check_is_supported +from arekit.common.labels.base import Label +from arekit.common.opinions.collection import OpinionCollection +from arekit.contrib.utils.evaluation.results.three_class_prf import ThreeClassPrecRecallF1EvalResult + + +class ThreeClassOpinionEvaluator(BaseEvaluator): + """ This evaluator is limitied and applied for OpinionCollections. + """ + + def __init__(self, label1, label2, no_label): + """ Since we consider that NEU labels are not a part of the result data. + Therefore if something missed in results then it suppose to be correct classified. + """ + assert(isinstance(label1, Label)) + assert(isinstance(label2, Label)) + assert(isinstance(no_label, Label)) + comparator = OpinionBasedComparator(EvaluationModes.Extraction) + + self.__label1 = label1 + self.__label2 = label2 + self.__no_label = no_label + + super(ThreeClassOpinionEvaluator, self).__init__(comparator) + + def _calc_diff(self, etalon_data, test_data, is_label_supported): + assert(isinstance(etalon_data, OpinionCollection)) + assert(isinstance(test_data, OpinionCollection)) + + # Composing test opinion collection + # by combining already existed with + # the neutrally labeled opinions. + test_opins_expanded = test_data.copy() + + for opinion in etalon_data: + # We keep only those opinions that were not + # presented in test and has neutral label + + check_is_supported(label=opinion.Sentiment, is_label_supported=is_label_supported) + + if not test_opins_expanded.has_synonymous_opinion(opinion) and opinion.Sentiment == self.__no_label: + test_opins_expanded.add_opinion(opinion) + + return super(ThreeClassOpinionEvaluator, self)._calc_diff(etalon_data=etalon_data, + test_data=test_opins_expanded, + is_label_supported=is_label_supported) + + def _create_eval_result(self): + return ThreeClassPrecRecallF1EvalResult(label1=self.__label1, + label2=self.__label2, + label3=self.__no_label, + get_item_label_func=lambda opinion: opinion.Sentiment) diff --git a/arekit/contrib/utils/evaluation/evaluators/three_class.py b/arekit/contrib/utils/evaluation/evaluators/three_class.py index f943827b..dca6d871 100644 --- a/arekit/contrib/utils/evaluation/evaluators/three_class.py +++ b/arekit/contrib/utils/evaluation/evaluators/three_class.py @@ -1,55 +1,23 @@ -from arekit.common.evaluation.comparators.opinions import OpinionBasedComparator from arekit.common.evaluation.evaluators.base import BaseEvaluator -from arekit.common.evaluation.evaluators.modes import EvaluationModes -from arekit.common.evaluation.evaluators.utils import check_is_supported from arekit.common.labels.base import Label -from arekit.common.opinions.collection import OpinionCollection from arekit.contrib.utils.evaluation.results.three_class_prf import ThreeClassPrecRecallF1EvalResult -class ThreeClassOpinionEvaluator(BaseEvaluator): - """ This evaluator is limitied and applied for OpinionCollections. - """ +class TwoClassEvaluator(BaseEvaluator): - def __init__(self, label1, label2, no_label): - """ Since we consider that NEU labels are not a part of the result data. - Therefore if something missed in results then it suppose to be correct classified. - """ + def __init__(self, comparator, label1, label2, label3, get_item_label_func): assert(isinstance(label1, Label)) assert(isinstance(label2, Label)) - assert(isinstance(no_label, Label)) - comparator = OpinionBasedComparator(EvaluationModes.Extraction) + assert(isinstance(label3, Label)) + assert(callable(get_item_label_func)) + super(TwoClassEvaluator, self).__init__(comparator) self.__label1 = label1 self.__label2 = label2 - self.__no_label = no_label - - super(ThreeClassOpinionEvaluator, self).__init__(comparator) - - def _calc_diff(self, etalon_data, test_data, is_label_supported): - assert(isinstance(etalon_data, OpinionCollection)) - assert(isinstance(test_data, OpinionCollection)) - - # Composing test opinion collection - # by combining already existed with - # the neutrally labeled opinions. - test_opins_expanded = test_data.copy() - - for opinion in etalon_data: - # We keep only those opinions that were not - # presented in test and has neutral label - - check_is_supported(label=opinion.Sentiment, is_label_supported=is_label_supported) - - if not test_opins_expanded.has_synonymous_opinion(opinion) and opinion.Sentiment == self.__no_label: - test_opins_expanded.add_opinion(opinion) - - return super(ThreeClassOpinionEvaluator, self)._calc_diff(etalon_data=etalon_data, - test_data=test_opins_expanded, - is_label_supported=is_label_supported) + self.__get_item_label_func = get_item_label_func def _create_eval_result(self): return ThreeClassPrecRecallF1EvalResult(label1=self.__label1, label2=self.__label2, - no_label=self.__no_label, - get_item_label_func=lambda opinion: opinion.Sentiment) + label3=self.__label2, + get_item_label_func=self.__get_item_label_func) diff --git a/arekit/contrib/utils/evaluation/results/three_class_prf.py b/arekit/contrib/utils/evaluation/results/three_class_prf.py index 3f48d138..f61144e0 100644 --- a/arekit/contrib/utils/evaluation/results/three_class_prf.py +++ b/arekit/contrib/utils/evaluation/results/three_class_prf.py @@ -26,15 +26,15 @@ class ThreeClassPrecRecallF1EvalResult(BaseEvalResult): C_F1_NEU = 'f1_neu' C_F1_MICRO = 'f1_micro' - def __init__(self, label1, label2, no_label, get_item_label_func): + def __init__(self, label1, label2, label3, get_item_label_func): assert(isinstance(label1, Label)) assert(isinstance(label2, Label)) - assert(isinstance(no_label, Label)) + assert(isinstance(label3, Label)) assert(callable(get_item_label_func)) self.__pos_label = label1 self.__neg_label = label2 - self.__neu_label = no_label + self.__neu_label = label3 self.__get_item_label_func = get_item_label_func super(ThreeClassPrecRecallF1EvalResult, self).__init__(