4.metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a2b862ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(121,\n",
       " ['accuracy',\n",
       "  'bertscore',\n",
       "  'bleu',\n",
       "  'bleurt',\n",
       "  'brier_score',\n",
       "  'cer',\n",
       "  'character',\n",
       "  'charcut_mt',\n",
       "  'chrf',\n",
       "  'code_eval',\n",
       "  'comet',\n",
       "  'competition_math',\n",
       "  'coval',\n",
       "  'cuad',\n",
       "  'exact_match',\n",
       "  'f1',\n",
       "  'frugalscore',\n",
       "  'glue',\n",
       "  'google_bleu',\n",
       "  'indic_glue',\n",
       "  'mae',\n",
       "  'mahalanobis',\n",
       "  'mape',\n",
       "  'mase',\n",
       "  'matthews_correlation',\n",
       "  'mauve',\n",
       "  'mean_iou',\n",
       "  'meteor',\n",
       "  'mse',\n",
       "  'nist_mt',\n",
       "  'pearsonr',\n",
       "  'perplexity',\n",
       "  'poseval',\n",
       "  'precision',\n",
       "  'recall',\n",
       "  'rl_reliability',\n",
       "  'roc_auc',\n",
       "  'rouge',\n",
       "  'sacrebleu',\n",
       "  'sari',\n",
       "  'seqeval',\n",
       "  'smape',\n",
       "  'spearmanr',\n",
       "  'squad',\n",
       "  'squad_v2',\n",
       "  'super_glue',\n",
       "  'ter',\n",
       "  'trec_eval',\n",
       "  'wer',\n",
       "  'wiki_split',\n",
       "  'xnli',\n",
       "  'xtreme_s',\n",
       "  'BucketHeadP65/confusion_matrix',\n",
       "  'BucketHeadP65/roc_curve',\n",
       "  'Drunper/metrica_tesi',\n",
       "  'Felipehonorato/my_metric',\n",
       "  'GMFTBY/dailydialog_evaluate',\n",
       "  'GMFTBY/dailydialogevaluate',\n",
       "  'JP-SystemsX/nDCG',\n",
       "  'Josh98/nl2bash_m',\n",
       "  'KevinSpaghetti/accuracyk',\n",
       "  'NCSOFT/harim_plus',\n",
       "  'NikitaMartynov/spell-check-metric',\n",
       "  'NimaBoscarino/weat',\n",
       "  'Ochiroo/rouge_mn',\n",
       "  'Vertaix/vendiscore',\n",
       "  'Viona/infolm',\n",
       "  'Vlasta/pr_auc',\n",
       "  'abdusah/aradiawer',\n",
       "  'abidlabs/mean_iou',\n",
       "  'abidlabs/mean_iou2',\n",
       "  'angelina-wang/directional_bias_amplification',\n",
       "  'anz2/iliauniiccocrevaluation',\n",
       "  'bstrai/classification_report',\n",
       "  'cakiki/ndcg',\n",
       "  'codeparrot/apps_metric',\n",
       "  'cpllab/syntaxgym',\n",
       "  'daiyizheng/valid',\n",
       "  'dvitel/codebleu',\n",
       "  'ecody726/bertscore',\n",
       "  'erntkn/dice_coefficient',\n",
       "  'giulio98/code_eval_outputs',\n",
       "  'giulio98/codebleu',\n",
       "  'gnail/cosine_similarity',\n",
       "  'gorkaartola/metric_for_tp_fp_samples',\n",
       "  'hack/test_metric',\n",
       "  'harshhpareek/bertscore',\n",
       "  'hpi-dhc/FairEval',\n",
       "  'idsedykh/codebleu',\n",
       "  'idsedykh/codebleu2',\n",
       "  'idsedykh/megaglue',\n",
       "  'idsedykh/metric',\n",
       "  'jordyvl/ece',\n",
       "  'jpxkqx/peak_signal_to_noise_ratio',\n",
       "  'jpxkqx/signal_to_reconstrution_error',\n",
       "  'jzm-mailchimp/joshs_second_test_metric',\n",
       "  'kaggle/ai4code',\n",
       "  'kaggle/amex',\n",
       "  'kashif/mape',\n",
       "  'kasmith/woodscore',\n",
       "  'kyokote/my_metric2',\n",
       "  'leslyarun/fbeta_score',\n",
       "  'loubnabnl/apps_metric2',\n",
       "  'lvwerra/accuracy_score',\n",
       "  'lvwerra/bary_score',\n",
       "  'lvwerra/test',\n",
       "  'mfumanelli/geometric_mean',\n",
       "  'mgfrantz/roc_auc_macro',\n",
       "  'ola13/precision_at_k',\n",
       "  'omidf/squad_precision_recall',\n",
       "  'posicube/mean_reciprocal_rank',\n",
       "  'ronaldahmed/nwentfaithfulness',\n",
       "  'sportlosos/sescore',\n",
       "  'transZ/sbert_cosine',\n",
       "  'transZ/test_parascore',\n",
       "  'xu1998hz/sescore',\n",
       "  'ybelkada/cocoevaluate',\n",
       "  'yonting/average_precision_score',\n",
       "  'yulong-me/yl_metric',\n",
       "  'yzha/ctc_eval',\n",
       "  'zbeloki/m2'])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import list_metrics\n",
    "\n",
    "#列出评价指标\n",
    "metrics_list = list_metrics()\n",
    "\n",
    "len(metrics_list), metrics_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d699a36e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Compute GLUE evaluation metric associated to each GLUE dataset.\n",
      "Args:\n",
      "    predictions: list of predictions to score.\n",
      "        Each translation should be tokenized into a list of tokens.\n",
      "    references: list of lists of references for each translation.\n",
      "        Each reference should be tokenized into a list of tokens.\n",
      "Returns: depending on the GLUE subset, one or several of:\n",
      "    \"accuracy\": Accuracy\n",
      "    \"f1\": F1 score\n",
      "    \"pearson\": Pearson Correlation\n",
      "    \"spearmanr\": Spearman Correlation\n",
      "    \"matthews_correlation\": Matthew Correlation\n",
      "Examples:\n",
      "\n",
      "    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]\n",
      "    >>> references = [0, 1]\n",
      "    >>> predictions = [0, 1]\n",
      "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
      "    >>> print(results)\n",
      "    {'accuracy': 1.0}\n",
      "\n",
      "    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'\n",
      "    >>> references = [0, 1]\n",
      "    >>> predictions = [0, 1]\n",
      "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
      "    >>> print(results)\n",
      "    {'accuracy': 1.0, 'f1': 1.0}\n",
      "\n",
      "    >>> glue_metric = datasets.load_metric('glue', 'stsb')\n",
      "    >>> references = [0., 1., 2., 3., 4., 5.]\n",
      "    >>> predictions = [0., 1., 2., 3., 4., 5.]\n",
      "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
      "    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})\n",
      "    {'pearson': 1.0, 'spearmanr': 1.0}\n",
      "\n",
      "    >>> glue_metric = datasets.load_metric('glue', 'cola')\n",
      "    >>> references = [0, 1]\n",
      "    >>> predictions = [0, 1]\n",
      "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
      "    >>> print(results)\n",
      "    {'matthews_correlation': 1.0}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_metric\n",
    "\n",
    "#加载一个评价指标\n",
    "metric = load_metric('glue', 'mrpc')\n",
    "\n",
    "print(metric.inputs_description)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5711e3ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.6666666666666666, 'f1': 0.6666666666666666}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#计算一个评价指标\n",
    "predictions = [0, 1, 0]\n",
    "references = [0, 1, 1]\n",
    "\n",
    "final_score = metric.compute(predictions=predictions, references=references)\n",
    "\n",
    "final_score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}