4-predict-on-unseen-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "CUDA not available - defaulting to CPU. Note: This module is much faster with a GPU.\n"
     ]
    }
   ],
   "source": [
    "import pickle, os, easyocr\n",
    "from pathlib import Path\n",
    "reader = easyocr.Reader(['en'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_whitespace(line_lower):\n",
    "    return  \" \".join(line_lower.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import RegexpTokenizer\n",
    "\n",
    "def remove_punct(text):\n",
    "    \n",
    "    tokenizer = RegexpTokenizer(r\"\\w+\")\n",
    "    lst=tokenizer.tokenize(' '.join(text))\n",
    "    return lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "en_stopwords = stopwords.words('english')\n",
    "\n",
    "def remove_stopwords(text):\n",
    "    result = []\n",
    "    for token in text:\n",
    "        if token not in en_stopwords:\n",
    "            result.append(token)\n",
    "            \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import word_tokenize,pos_tag\n",
    "\n",
    "def lemmatization(text):\n",
    "    \n",
    "    result=[]\n",
    "    wordnet = WordNetLemmatizer()\n",
    "    for token,tag in pos_tag(text):\n",
    "        pos=tag[0].lower()\n",
    "        \n",
    "        if pos not in ['a', 'r', 'n', 'v']:\n",
    "            pos='n'\n",
    "            \n",
    "        result.append(wordnet.lemmatize(token,pos))\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def image_to_text(image_path):\n",
    "\n",
    "\n",
    "    if image_path.endswith('.png'):\n",
    "        ocr_result = reader.readtext(str(image_path), batch_size=16)\n",
    "\n",
    "        ocr_page = \"\"\n",
    "        for bbox, word, confidance in ocr_result:\n",
    "            ocr_page += word + \" \"\n",
    "\n",
    "        with Path(str(image_path).replace('.png', '.txt')).open('w') as f:\n",
    "            line_lower = ocr_page.lower()\n",
    "            line_without_space = remove_whitespace(line_lower)\n",
    "            token_list = word_tokenize(line_without_space)\n",
    "            clean_text = remove_stopwords(token_list)\n",
    "            no_punct = remove_punct(clean_text)\n",
    "            lemma_text = lemmatization(no_punct)\n",
    "            clean_text = \" \".join(lemma_text)\n",
    "\n",
    "            # json.dump(clean_text[1:-1], f)\n",
    "            # text_file = open(os.path.join(text_folder, \"data.txt\"), 'w')\n",
    "            f.write(clean_text)\n",
    "            f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_model = pickle.load(open('model.1.0.0.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_class(probability_array, prediction):\n",
    "\n",
    "    ClassToBeReturned = \"Cound not classify\"\n",
    "    if max(probability_array.tolist()) > 0.9:\n",
    "\n",
    "        if prediction[0] == 1:\n",
    "            ClassToBeReturned = 'W9 form'\n",
    "        elif prediction[0] == 2:\n",
    "            ClassToBeReturned = 'iBAN'\n",
    "        elif prediction[0] == 3:\n",
    "            ClassToBeReturned = 'ISO certificate'\n",
    "\n",
    "\n",
    "    return ClassToBeReturned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.join(os.getcwd(), 'Test Images')\n",
    "image_path = os.path.join(path, 'W9-200.png')\n",
    "text_path = image_path.replace('.png', '.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_to_text(image_path)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Predict method predicts \n",
    "1: W9 | 2: iBAN | 3: ISO\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Probability array:  [1. 0. 0.]\n",
      "Prediction:  [1]\n",
      "Predicted Class:  W9 form\n",
      "Wall time: 24.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open(text_path, 'r') as f:\n",
    "    text = f.readlines()\n",
    "    probability_array = loaded_model.predict_proba(text)[0]\n",
    "    prediction = loaded_model.predict(text)\n",
    "    predicted_class = predict_class(probability_array, prediction)\n",
    "    print(\"The Probability array: \", probability_array)\n",
    "    print(\"Prediction: \", prediction)\n",
    "    print('Predicted Class: ', predicted_class)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "9650cb4e16cdd4a8e8e2d128bf38d875813998db22a3c986335f89e0cb4d7bb2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}