compress_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from icl.analysis.compress import CompressArgs\n",
    "from sklearn.metrics import accuracy_score\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_label(y):\n",
    "    return y.predictions[0].argmax(-1)\n",
    "\n",
    "def get_logits(y):\n",
    "    if y.predictions[2].shape[-1] > 30000:\n",
    "        return y.predictions[2]\n",
    "    else:\n",
    "        return y.predictions[3]\n",
    "\n",
    "def get_topk(y, k):\n",
    "    logits = get_logits(y)\n",
    "    indices = np.argpartition(logits, -k,axis=1)[:,-k:]\n",
    "    return indices\n",
    "\n",
    "def get_sorted_topk(y, k):\n",
    "    logits = get_logits(y)\n",
    "    indices = torch.topk(torch.tensor(logits),k)[1].numpy()\n",
    "    return indices\n",
    "\n",
    "def jaccard(a,b):\n",
    "    scores = []\n",
    "    for single_a, single_b in zip(a,b):\n",
    "        set_a = set(single_a)\n",
    "        set_b = set(single_b)\n",
    "        score = len(set_a.intersection(set_b))/len(set_a.union(set_b))\n",
    "        scores.append(score)\n",
    "    return np.array(scores).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "from icl.utils.load_huggingface_dataset import load_huggingface_dataset_train_and_test\n",
    "import warnings\n",
    "\n",
    "def get_true_label(dataset, seed, actual_sample_size):\n",
    "    test_sample = dataset['test'].shuffle(seed=seed).select(range(actual_sample_size))\n",
    "    labels = np.array(test_sample['label'])\n",
    "    return labels\n",
    "\n",
    "def calculate_average_scores(seeds, task_name,actual_sample_size=1000):\n",
    "    scores = []\n",
    "    accs = []\n",
    "    jaccards = []\n",
    "    dataset = load_huggingface_dataset_train_and_test(task_name)\n",
    "    if len(dataset['test']) < actual_sample_size:\n",
    "        warnings.warn(\n",
    "            f\"sample_size: {actual_sample_size} is larger than test set size: {len(dataset['test'])},\"\n",
    "            f\"actual_sample_size is {len(dataset['test'])}\")\n",
    "        actual_sample_size = len(dataset['test'])\n",
    "    \n",
    "    for seed in tqdm(seeds):\n",
    "        label = get_true_label(dataset=dataset,seed=seed,actual_sample_size=actual_sample_size)\n",
    "        args = CompressArgs(task_name=task_name, seeds=[seed],model_name='gpt-j-6b')\n",
    "        y1, y2, y3, y4 = args.load_result()[0]\n",
    "        label1, label2, label3, label4 = [get_label(_) for _ in [y1, y2, y3, y4]]\n",
    "        score1 = accuracy_score(label2, label1) # Hidden_anchor\n",
    "        score2 = accuracy_score(label2, label3) # Text_anchor\n",
    "        score3 = accuracy_score(label2, label4) # Hidden_random\n",
    "        scores.append((score1, score2, score3)) # Label Loyalty\n",
    "\n",
    "        acc0  = accuracy_score(label, label2)\n",
    "        acc1 = accuracy_score(label, label1)\n",
    "        acc2 = accuracy_score(label, label3)\n",
    "        acc3 = accuracy_score(label, label4)\n",
    "        accs.append((acc0,acc1, acc2, acc3)) # Acc.\n",
    "\n",
    "        jaccard_1 = jaccard(get_topk(y1, 5), get_topk(y2, 5))\n",
    "        jaccard_2 = jaccard(get_topk(y3, 5), get_topk(y2, 5))\n",
    "        jaccard_3 = jaccard(get_topk(y4, 5), get_topk(y2, 5))\n",
    "        jaccards.append((jaccard_1, jaccard_2, jaccard_3)) # Word Loyalty\n",
    "\n",
    "    average_scores = np.mean(scores, axis=0)\n",
    "    average_accs = np.mean(accs, axis=0)\n",
    "    average_jaccards = np.mean(jaccards, axis=0)\n",
    "    return average_scores, average_accs, average_jaccards\n",
    "\n",
    "seeds = [42, 43, 44, 45, 46]\n",
    "average_scores = calculate_average_scores(seeds, 'agnews')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.16 ('bbtv2')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "fa84ff37f7354d5baacf3f95c54ec9bb9436f05eafb6bc27ab368dac8f7f3b02"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}