kenlm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9a293c6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>split</th>\n",
       "      <th>path</th>\n",
       "      <th>durs</th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000005f3362c</td>\n",
       "      <td>ও বলেছে আপনার ঠিকানা!</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/000005f33...</td>\n",
       "      <td>1.116</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/000005f33...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00001dddd002</td>\n",
       "      <td>কোন মহান রাষ্ট্রের নাগরিক হতে চাও?</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00001dddd...</td>\n",
       "      <td>2.448</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00001dddd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00001e0bc131</td>\n",
       "      <td>আমি তোমার কষ্টটা বুঝছি, কিন্তু এটা সঠিক পথ না।</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00001e0bc...</td>\n",
       "      <td>4.716</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00001e0bc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000024b3d810</td>\n",
       "      <td>নাচ শেষ হওয়ার পর সকলে শরীর ধুয়ে একসঙ্গে ভোজন...</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/000024b3d...</td>\n",
       "      <td>7.452</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/000024b3d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000028220ab3</td>\n",
       "      <td>হুমম, ওহ হেই, দেখো।</td>\n",
       "      <td>train</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/000028220...</td>\n",
       "      <td>2.160</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/000028220...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                                           sentence  split  \\\n",
       "0  000005f3362c                              ও বলেছে আপনার ঠিকানা!  train   \n",
       "1  00001dddd002                 কোন মহান রাষ্ট্রের নাগরিক হতে চাও?  train   \n",
       "2  00001e0bc131     আমি তোমার কষ্টটা বুঝছি, কিন্তু এটা সঠিক পথ না।  train   \n",
       "3  000024b3d810  নাচ শেষ হওয়ার পর সকলে শরীর ধুয়ে একসঙ্গে ভোজন...  train   \n",
       "4  000028220ab3                                হুমম, ওহ হেই, দেখো।  train   \n",
       "\n",
       "                                                path   durs  \\\n",
       "0  /home/ubuntu/bengali/data/train_mp3s/000005f33...  1.116   \n",
       "1  /home/ubuntu/bengali/data/train_mp3s/00001dddd...  2.448   \n",
       "2  /home/ubuntu/bengali/data/train_mp3s/00001e0bc...  4.716   \n",
       "3  /home/ubuntu/bengali/data/train_mp3s/000024b3d...  7.452   \n",
       "4  /home/ubuntu/bengali/data/train_mp3s/000028220...  2.160   \n",
       "\n",
       "                                            filename  \n",
       "0  /home/ubuntu/bengali/data/train_mp3s/000005f33...  \n",
       "1  /home/ubuntu/bengali/data/train_mp3s/00001dddd...  \n",
       "2  /home/ubuntu/bengali/data/train_mp3s/00001e0bc...  \n",
       "3  /home/ubuntu/bengali/data/train_mp3s/000024b3d...  \n",
       "4  /home/ubuntu/bengali/data/train_mp3s/000028220...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_parquet('../data/train_duration.parquet')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "910f7ff7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(963636, 6)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3135cf1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('text.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(df['sentence'].tolist()))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2e495b35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kenlm/build/bin/lmplz: error while loading shared libraries: libboost_program_options.so.1.74.0: cannot open shared object file: No such file or directory\n",
      "/home/ubuntu/bengali/speech-to-text/kenlm/util/file.cc:76 in int util::OpenReadOrThrow(const char*) threw ErrnoException because `-1 == (ret = open(name, 00))'.\n",
      "No such file or directory while opening out.arpa\n",
      "ERROR\n"
     ]
    }
   ],
   "source": [
    "!kenlm/build/bin/lmplz --text text.txt --arpa out.arpa -o 4 --prune 0 1 1\n",
    "!kenlm/build/bin/build_binary -q 8 -b 7 -a 256 trie out.arpa out.trie.klm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "af28a34a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 154M\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu  55K Sep 11 05:22  Inference-Trial-0.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu 2.2K Sep  3 15:55  README.md\r\n",
      "-rw-r--r--  1 ubuntu ubuntu  915 Sep  7 11:31  audio-augmentation.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu 9.5M Sep 10 14:30  bengali-speech-eda.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu 767K Sep 10 08:41  check-unknown.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu  13K Sep 10 02:29  explore-vocab.ipynb\r\n",
      "-rw-r--r--  1 ubuntu ubuntu 7.6K Sep  6 04:01  inference-ariff.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu  32K Sep  3 15:55  inference-validation.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu  10K Sep  3 15:55  inference.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu  232 Sep  3 15:55  install-kenlm.sh\r\n",
      "drwxr-xr-x  9 ubuntu ubuntu 4.0K Sep 11 07:43  kenlm\r\n",
      "drwxr-xr-x  2 ubuntu ubuntu 4.0K Sep 12 00:41 'kenlm 4gram'\r\n",
      "drwxr-xr-x  2 ubuntu ubuntu 4.0K Sep 11 08:37  kenlm-arpa\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu 2.4K Sep 11 07:37  kenlm-test.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu  36K Sep 12 00:42  kenlm.ipynb\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu 3.3K Sep  3 15:55  prepare-data.ipynb\r\n",
      "-rw-r--r--  1 ubuntu ubuntu 134M Sep 12 00:40  text.txt\r\n",
      "-rw-r--r--  1 ubuntu ubuntu 9.9M Sep 11 02:24 'the problem of overfitting butter chicken.ipynb'\r\n",
      "drwxrwxrwx 12 ubuntu ubuntu 4.0K Sep 11 07:36  training\r\n",
      "-rwxrwxrwx  1 ubuntu ubuntu 1.1K Sep  4 10:30  vocab.json\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "26bf1728",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\data\\\r\n",
      "ngram 1=235764\r\n",
      "ngram 2=1811037\r\n",
      "ngram 3=2843169\r\n",
      "ngram 4=2909477\r\n",
      "\r\n",
      "\\1-grams:\r\n",
      "-6.338945\t<unk>\t0\r\n",
      "0\t<s>\t-1.2010338\r\n",
      "-1.6758004\t</s>\t0\r\n",
      "-2.3550007\tও\t-0.28947067\r\n",
      "-4.3527513\tবলেছে\t-0.2332048\r\n",
      "-3.303406\tআপনার\t-0.27609196\r\n",
      "-6.216548\tঠিকানা!\t-0.09186897\r\n",
      "-3.0924196\tকোন\t-0.31199992\r\n",
      "-4.1793427\tমহান\t-0.15257786\r\n",
      "-4.1793427\tরাষ্ট্রের\t-0.15116856\r\n",
      "-4.01065\tনাগরিক\t-0.24162138\r\n",
      "-2.9815621\tহতে\t-0.65213895\r\n",
      "-4.4986434\tচাও?\t-1.6840796\r\n"
     ]
    }
   ],
   "source": [
    "!head -20 out.arpa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "501b2ea2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# !pip3 install pyctcdecode pypi-kenlm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fc2ad211",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<pad>',\n",
       " '<s>',\n",
       " '</s>',\n",
       " '<unk>',\n",
       " '|',\n",
       " 'া',\n",
       " 'র',\n",
       " 'ে',\n",
       " '্',\n",
       " 'ি',\n",
       " 'ন',\n",
       " 'ক',\n",
       " 'য',\n",
       " 'ব',\n",
       " 'ত',\n",
       " 'স',\n",
       " 'ল',\n",
       " 'প',\n",
       " '়',\n",
       " 'ম',\n",
       " 'ট',\n",
       " 'ু',\n",
       " 'দ',\n",
       " 'এ',\n",
       " 'হ',\n",
       " 'ো',\n",
       " 'জ',\n",
       " 'গ',\n",
       " 'শ',\n",
       " 'ছ',\n",
       " 'ী',\n",
       " 'ই',\n",
       " 'থ',\n",
       " 'ভ',\n",
       " 'অ',\n",
       " 'আ',\n",
       " 'ং',\n",
       " 'ষ',\n",
       " 'চ',\n",
       " 'ড',\n",
       " 'ধ',\n",
       " 'ও',\n",
       " 'ণ',\n",
       " 'খ',\n",
       " 'উ',\n",
       " 'ফ',\n",
       " '\"',\n",
       " 'ূ',\n",
       " '0',\n",
       " 'ৃ',\n",
       " '।',\n",
       " 'ঘ',\n",
       " '1',\n",
       " 'ঠ',\n",
       " 'ঁ',\n",
       " '-',\n",
       " 'ঞ',\n",
       " '2',\n",
       " 'ৈ',\n",
       " 'ঙ',\n",
       " 'ৌ',\n",
       " '9',\n",
       " '5',\n",
       " '4',\n",
       " 'ৎ',\n",
       " '3',\n",
       " 'ঝ',\n",
       " '০',\n",
       " '6',\n",
       " '8',\n",
       " ',',\n",
       " \"'\",\n",
       " '7',\n",
       " 'a',\n",
       " '১',\n",
       " '.',\n",
       " 'ঃ',\n",
       " 'c',\n",
       " '৪',\n",
       " ':',\n",
       " '২',\n",
       " '৯',\n",
       " 'ঐ',\n",
       " 't',\n",
       " 's',\n",
       " '/',\n",
       " 'm',\n",
       " 'p',\n",
       " '৬',\n",
       " 'i',\n",
       " '৫',\n",
       " '৮',\n",
       " 'n',\n",
       " '৩',\n",
       " 'ঢ',\n",
       " '৭',\n",
       " 'u',\n",
       " 'o',\n",
       " 'r',\n",
       " 'd',\n",
       " 'h',\n",
       " 'e',\n",
       " 'v',\n",
       " 'l',\n",
       " 'g',\n",
       " 'b',\n",
       " 'w',\n",
       " 'ঈ',\n",
       " ';',\n",
       " 'y',\n",
       " 'k',\n",
       " 'x',\n",
       " '\\u200d',\n",
       " '%',\n",
       " 'f',\n",
       " '!',\n",
       " 'ঔ',\n",
       " '¥',\n",
       " 'õ',\n",
       " 'ঊ',\n",
       " 'ঋ',\n",
       " '°',\n",
       " 'q',\n",
       " 'í',\n",
       " '+',\n",
       " 'ü',\n",
       " 'z',\n",
       " '[',\n",
       " ']',\n",
       " '£',\n",
       " '৷',\n",
       " '?',\n",
       " 'j',\n",
       " 'ú',\n",
       " 'á',\n",
       " '—']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab.json') as fopen:\n",
    "    vocab = json.load(fopen)\n",
    "    vocab = {v: k for k, v in vocab.items()}\n",
    "    \n",
    "vocab = [vocab[i] for i in range(len(vocab))]\n",
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "db9581e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading the LM will be faster if you build a binary file.\n",
      "Reading /home/ubuntu/bengali/aisyah/out.arpa\n",
      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
      "****************************************************************************************************\n",
      "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n"
     ]
    }
   ],
   "source": [
    "import kenlm\n",
    "from pyctcdecode import build_ctcdecoder\n",
    "\n",
    "lm = 'out.arpa'\n",
    "kenlm_model = lm\n",
    "decoder = build_ctcdecoder(\n",
    "    vocab,\n",
    "    kenlm_model,\n",
    "    alpha=0.5,\n",
    "    beta=1.0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1c89c6a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-09-01 16:16:33,958] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-01 16:16:38.685764: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-09-01 16:16:40.024829: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from transformers import (\n",
    "    Wav2Vec2ForCTC,\n",
    "    Wav2Vec2Processor,\n",
    "    Wav2Vec2CTCTokenizer,\n",
    "    Wav2Vec2FeatureExtractor\n",
    ") \n",
    "\n",
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoFeatureExtractor,\n",
    "    AutoModelForCTC,\n",
    "    AutoProcessor,\n",
    "    AutoTokenizer,\n",
    "    HfArgumentParser,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    "    Wav2Vec2Processor,\n",
    "    set_seed,\n",
    ")\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "de2ef719",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "import random\n",
    "import pandas as pd\n",
    "import torch\n",
    "from transformers import AutoTokenizer\n",
    "from typing import Mapping, Tuple\n",
    "import librosa\n",
    "# import en_core_web_sm\n",
    "\n",
    "\n",
    "import librosa\n",
    "\n",
    "class BengaliDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, df, processor):\n",
    "        self.df = df\n",
    "        self.processor = processor\n",
    "\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        audio_path = self.df.loc[idx]['path']\n",
    "        audio_array = self.read_audio(audio_path)\n",
    "        \n",
    "        inputs = self.processor(\n",
    "            audio_array,\n",
    "            sampling_rate=16000,\n",
    "            return_tensors='pt'  \n",
    "        )\n",
    "        \n",
    "        with self.processor.as_target_processor():\n",
    "            labels = self.processor(self.df.loc[idx]['sentence']).input_ids\n",
    "        \n",
    "        return {'input_values': inputs['input_values'][0], 'labels': labels}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def read_audio(self, mp3_path):\n",
    "        target_sr = 16000  # Set the target sampling rate\n",
    "        \n",
    "        audio, sr = librosa.load(mp3_path, sr=None)  # Load with original sampling rate\n",
    "        audio_array = librosa.resample(audio, orig_sr=sr, target_sr=target_sr)\n",
    "        \n",
    "        return audio_array\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "07c7f331",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint-16000  checkpoint-16400  runs\r\n"
     ]
    }
   ],
   "source": [
    "!ls /home/ubuntu/bengali/aisyah/training/mms-1b/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b7eb0ad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "processor = AutoProcessor.from_pretrained(\"/home/ubuntu/bengali/aisyah/training/mms-1b/checkpoint-16400\")\n",
    "model = Wav2Vec2ForCTC.from_pretrained(\"/home/ubuntu/bengali/aisyah/training/mms-1b/checkpoint-16400\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5f37fc07",
   "metadata": {},
   "outputs": [],
   "source": [
    "processor.tokenizer.set_target_lang(\"ben\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "68f261f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Wav2Vec2CTCTokenizer(\n",
    "    \"vocab.json\", \n",
    "    unk_token=\"<unk>\",\n",
    "    pad_token=\"<pad>\",\n",
    "    word_delimiter_token=\"|\"\n",
    ")\n",
    "\n",
    "feature_extractor = Wav2Vec2FeatureExtractor(\n",
    "    feature_size=1, \n",
    "    sampling_rate=16000, \n",
    "    padding_value=0.0, \n",
    "    do_normalize=True, \n",
    "    return_attention_mask=False\n",
    ")\n",
    "\n",
    "# valid_ds = BengaliDataset(val,processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "cda43c09",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('../data/train.csv')\n",
    "\n",
    "test = test[test['split'] =='valid'].reset_index(drop=True)\n",
    "\n",
    "test['path'] = test['id'].apply(lambda x: os.path.join('/home/ubuntu/bengali/data/train_mp3s', x+'.mp3'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "2a778c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = test[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "0b522eea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>split</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000e711c2b1</td>\n",
       "      <td>তিনি এবং তাঁর মা তাদের পৈতৃক বাড়িতে থেকে প্রত...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/0000e711c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00036c2a2d9d</td>\n",
       "      <td>কৃত্তিবাস রামায়ণ-বহির্ভূত অনেক গল্প এই অনুবাদ...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00036c2a2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00065e317123</td>\n",
       "      <td>তিনি তার সুশৃঙ্খল সামরিক বাহিনী এবং সুগঠিত শাস...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00065e317...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00065f40df52</td>\n",
       "      <td>তিনি বিজয়নগর সাম্রাজ্যের বিরুদ্ধে এবং বিজাপুর...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00065f40d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0009b022c8ea</td>\n",
       "      <td>এটি মূলত একটি মরুময় অঞ্চল।</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/0009b022c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                                           sentence  split  \\\n",
       "0  0000e711c2b1  তিনি এবং তাঁর মা তাদের পৈতৃক বাড়িতে থেকে প্রত...  valid   \n",
       "1  00036c2a2d9d  কৃত্তিবাস রামায়ণ-বহির্ভূত অনেক গল্প এই অনুবাদ...  valid   \n",
       "2  00065e317123  তিনি তার সুশৃঙ্খল সামরিক বাহিনী এবং সুগঠিত শাস...  valid   \n",
       "3  00065f40df52  তিনি বিজয়নগর সাম্রাজ্যের বিরুদ্ধে এবং বিজাপুর...  valid   \n",
       "4  0009b022c8ea                        এটি মূলত একটি মরুময় অঞ্চল।  valid   \n",
       "\n",
       "                                                path  \n",
       "0  /home/ubuntu/bengali/data/train_mp3s/0000e711c...  \n",
       "1  /home/ubuntu/bengali/data/train_mp3s/00036c2a2...  \n",
       "2  /home/ubuntu/bengali/data/train_mp3s/00065e317...  \n",
       "3  /home/ubuntu/bengali/data/train_mp3s/00065f40d...  \n",
       "4  /home/ubuntu/bengali/data/train_mp3s/0009b022c...  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "09586498",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ds = BengaliDataset(test,processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "671d5721",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_cer(actual, hyp):\n",
    "    \"\"\"\n",
    "    Calculate CER using `python-Levenshtein`.\n",
    "    \"\"\"\n",
    "    import Levenshtein as Lev\n",
    "\n",
    "    actual = actual.replace(' ', '')\n",
    "    hyp = hyp.replace(' ', '')\n",
    "    return Lev.distance(actual, hyp) / len(actual)\n",
    "\n",
    "def calculate_wer(actual, hyp):\n",
    "    \"\"\"\n",
    "    Calculate WER using `python-Levenshtein`.\n",
    "    \"\"\"\n",
    "    import Levenshtein as Lev\n",
    "\n",
    "    b = set(actual.split() + hyp.split())\n",
    "    word2char = dict(zip(b, range(len(b))))\n",
    "\n",
    "    w1 = [chr(word2char[w]) for w in actual.split()]\n",
    "    w2 = [chr(word2char[w]) for w in hyp.split()]\n",
    "\n",
    "    return Lev.distance(''.join(w1), ''.join(w2)) / len(actual.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "70dca781",
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "be4dd350",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>split</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000e711c2b1</td>\n",
       "      <td>তিনি এবং তাঁর মা তাদের পৈতৃক বাড়িতে থেকে প্রত...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/0000e711c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00036c2a2d9d</td>\n",
       "      <td>কৃত্তিবাস রামায়ণ-বহির্ভূত অনেক গল্প এই অনুবাদ...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00036c2a2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00065e317123</td>\n",
       "      <td>তিনি তার সুশৃঙ্খল সামরিক বাহিনী এবং সুগঠিত শাস...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00065e317...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00065f40df52</td>\n",
       "      <td>তিনি বিজয়নগর সাম্রাজ্যের বিরুদ্ধে এবং বিজাপুর...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/00065f40d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0009b022c8ea</td>\n",
       "      <td>এটি মূলত একটি মরুময় অঞ্চল।</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/0009b022c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>088042fde886</td>\n",
       "      <td>এর এক মাস পরে তিনি মারা যান।</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/088042fde...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0880afd89c6c</td>\n",
       "      <td>কিন্তু বর্তমানে কালির ব্যবহার সব কিছু সহজ করে ...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/0880afd89...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0882b463c8fc</td>\n",
       "      <td>কলেজটিতে একটি গ্রন্থাগার রয়েছে।</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/0882b463c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>08863cd41d51</td>\n",
       "      <td>হাতে গণনার জন্য প্রচুর সুত্র ও তথ্য সারণি সংবল...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/08863cd41...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>08881f513454</td>\n",
       "      <td>আশেপাশের গ্রামগুলিতে তাদের দেবতার প্রতি শ্রদ্ধ...</td>\n",
       "      <td>valid</td>\n",
       "      <td>/home/ubuntu/bengali/data/train_mp3s/08881f513...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id                                           sentence  split  \\\n",
       "0    0000e711c2b1  তিনি এবং তাঁর মা তাদের পৈতৃক বাড়িতে থেকে প্রত...  valid   \n",
       "1    00036c2a2d9d  কৃত্তিবাস রামায়ণ-বহির্ভূত অনেক গল্প এই অনুবাদ...  valid   \n",
       "2    00065e317123  তিনি তার সুশৃঙ্খল সামরিক বাহিনী এবং সুগঠিত শাস...  valid   \n",
       "3    00065f40df52  তিনি বিজয়নগর সাম্রাজ্যের বিরুদ্ধে এবং বিজাপুর...  valid   \n",
       "4    0009b022c8ea                        এটি মূলত একটি মরুময় অঞ্চল।  valid   \n",
       "..            ...                                                ...    ...   \n",
       "995  088042fde886                       এর এক মাস পরে তিনি মারা যান।  valid   \n",
       "996  0880afd89c6c  কিন্তু বর্তমানে কালির ব্যবহার সব কিছু সহজ করে ...  valid   \n",
       "997  0882b463c8fc                   কলেজটিতে একটি গ্রন্থাগার রয়েছে।  valid   \n",
       "998  08863cd41d51  হাতে গণনার জন্য প্রচুর সুত্র ও তথ্য সারণি সংবল...  valid   \n",
       "999  08881f513454  আশেপাশের গ্রামগুলিতে তাদের দেবতার প্রতি শ্রদ্ধ...  valid   \n",
       "\n",
       "                                                  path  \n",
       "0    /home/ubuntu/bengali/data/train_mp3s/0000e711c...  \n",
       "1    /home/ubuntu/bengali/data/train_mp3s/00036c2a2...  \n",
       "2    /home/ubuntu/bengali/data/train_mp3s/00065e317...  \n",
       "3    /home/ubuntu/bengali/data/train_mp3s/00065f40d...  \n",
       "4    /home/ubuntu/bengali/data/train_mp3s/0009b022c...  \n",
       "..                                                 ...  \n",
       "995  /home/ubuntu/bengali/data/train_mp3s/088042fde...  \n",
       "996  /home/ubuntu/bengali/data/train_mp3s/0880afd89...  \n",
       "997  /home/ubuntu/bengali/data/train_mp3s/0882b463c...  \n",
       "998  /home/ubuntu/bengali/data/train_mp3s/08863cd41...  \n",
       "999  /home/ubuntu/bengali/data/train_mp3s/08881f513...  \n",
       "\n",
       "[1000 rows x 4 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "64d4d1e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:12<00:00,  7.95it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "cer, wer, cer_lm, wer_lm = [], [], [], []\n",
    "\n",
    "for i in tqdm(range(len(test))):\n",
    "    input_dict = processor(test_ds[i]['input_values'], sampling_rate=16_000, return_tensors=\"pt\", padding=True)\n",
    "    for k in input_dict.keys():\n",
    "        input_dict[k] = input_dict[k].cuda()\n",
    "\n",
    "    logits = model(input_dict.input_values).logits\n",
    "\n",
    "    pred_ids = torch.argmax(logits, dim=-1)[0]\n",
    "    \n",
    "    out = decoder.decode_beams(logits[0].cpu().detach().numpy(), prune_history=True)\n",
    "    d_lm, lm_state, timesteps, logit_score, lm_score = out[0]\n",
    "    \n",
    "    cer.append(calculate_cer(test.loc[i][\"sentence\"], processor.decode(pred_ids)))\n",
    "    cer_lm.append(calculate_cer(test.loc[i][\"sentence\"], d_lm))\n",
    "    \n",
    "    wer.append(calculate_wer(test.loc[i][\"sentence\"], processor.decode(pred_ids)))\n",
    "    wer_lm.append(calculate_wer(test.loc[i][\"sentence\"], d_lm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b975c2ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "||k|u|c e n||g  -> ctc layer (classification) -> lineary layer -> sentence\n",
    "\n",
    "wav2vec + kenlm (ctc based model)\n",
    "encoder only model\n",
    "\n",
    "\n",
    "whisper =  encoder + decoder (seq2seq model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "8d2566f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.06901552451169052, 0.2674682686594451)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.mean(cer), np.mean(wer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "2549a6d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.037820660188787616, 0.11936407465083936)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(cer_lm), np.mean(wer_lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6e6b838",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}