ml-book-rag.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0567aeb4-10bb-4d93-b34f-d4d7001d3f08",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#!pip install -U transformers rank_bm25 sentence-transformers langchain langchain-experimental langchain-community \"unstructured[all-docs]\" --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3eec929a-3f23-423e-a8bf-504854e8dbd2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'4.38.2'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import transformers\n",
    "\n",
    "transformers.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a61ff498-a891-4f2c-87fd-c8c0091f3a50",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/torch/_utils.py:776: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
      "  return self.fget.__get__(instance, owner)()\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from functools import partial\n",
    "from unstructured.partition.auto import partition\n",
    "from unstructured.chunking import chunk_by_title\n",
    "from unstructured.documents.elements import Element, Text\n",
    "from unstructured.cleaners.core import clean\n",
    "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
    "from langchain.embeddings import SelfHostedHuggingFaceEmbeddings\n",
    "from langchain_experimental.text_splitter import SemanticChunker\n",
    "\n",
    "model_id = \"sentence-transformers/all-MiniLM-L6-v2\"\n",
    "model_kwargs = {\"device\": \"cuda\"}\n",
    "encode_kwargs = {\"normalize_embeddings\": True}\n",
    "\n",
    "model = HuggingFaceEmbeddings(\n",
    "    model_name=model_id, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6795eb0e-3cbf-4172-9125-9c38a34bd406",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text_splitter = SemanticChunker(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aa110baa-9dda-4d0e-85a0-4b2ac82e8e6b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "files = dict()\n",
    "\n",
    "files_path = Path(\"./data/\")\n",
    "\n",
    "for file in files_path.glob(\"*.pdf\"):\n",
    "    elements = partition(str(file.resolve()))\n",
    "    text_elements = chunk_by_title(elements)\n",
    "    \n",
    "    chunks = []\n",
    "\n",
    "    for element in text_elements:\n",
    "        element.apply(\n",
    "            partial(\n",
    "                clean,\n",
    "                bullets=True,\n",
    "                extra_whitespace=True,\n",
    "                dashes=True,\n",
    "                trailing_punctuation=True\n",
    "            )\n",
    "        )\n",
    "        chunks.append(element)\n",
    "    files[file.name] = chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6194d38f-8b42-47b9-9e82-5495825ddcb5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<unstructured.documents.elements.CompositeElement at 0x7fe1353fe440>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# After table of contents\n",
    "files[\"978-981-15-1967-3.pdf\"][117]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "552187a8-7bb7-43ce-b19c-b2b91abdf820",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<unstructured.documents.elements.CompositeElement at 0x7fe134abe5c0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Before index\n",
    "files[\"978-981-15-1967-3.pdf\"][-40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "20f3bca1-20c2-4062-8e23-27814da11db0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "book = [chunk.text for chunk in files[\"978-981-15-1967-3.pdf\"][117:-39]]\n",
    "book = \" \".join(book)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a2fa953e-5a0f-46fb-a49e-23af6dd71493",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "docs = text_splitter.create_documents([book])\n",
    "docs = [doc.page_content for doc in docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7fc34321-2645-46a9-9d84-148c9fb64645",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "questions = [\n",
    "    \"What is overfitting?\",\n",
    "    \"What is underfitting?\",\n",
    "    \"How do we test a models generalization error?\",\n",
    "    \"Why should training data points not be in the test set?\",\n",
    "    \"What is cross validation?\",\n",
    "    \"What are commonly used metrics to measure the performance of a model?\",\n",
    "    \"What is linear regression?\",\n",
    "    \"What are problems of linear models?\",\n",
    "    \"What is a decision tree?\",\n",
    "    \"What is the McCulloch–Pitts model?\",\n",
    "    \"What is a neural network?\",\n",
    "    \"How is a neural network optimized?\",\n",
    "    \"How does backpropagation work?\",\n",
    "    \"What is deep learning?\",\n",
    "    \"What is ensemble learning?\",\n",
    "    \"What is the goal of ensemble methods?\",\n",
    "    \"What is supervised learning?\"\n",
    "    \"What is unsupervised learning?\",\n",
    "    \"What is the difference between supervised and unsupervised learning?\",\n",
    "    \"What is the goal of clustering?\",\n",
    "    \"How is k-Nearest Neighbor trained?\",\n",
    "    \"Which algorithm can be used to reduce dimensions?\",\n",
    "    \"What is semi-supervised learning?\",\n",
    "    \"What is reinforcement learning?\",\n",
    "    \"What is an Markov Decision Process used for in reinforcement learning\",\n",
    "    \"How interacts an reinforcment learning agent with its environment?\",\n",
    "    \"What is the goal of an agent in reinforcement learning?\",\n",
    "    \"What is the Exploration-Exploitation dilemma?\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a723f96a-3029-498c-a52b-b96e46b9f967",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /home/chkei001/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "/opt/conda/lib/python3.10/site-packages/torch/_utils.py:776: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
      "  return self.fget.__get__(instance, owner)()\n"
     ]
    }
   ],
   "source": [
    "from model import DecoderModel\n",
    "from store import VectorStore\n",
    "\n",
    "db = VectorStore(\"sentence-transformers/all-MiniLM-L6-v2\", hybrid=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7c4d8c16-43da-491e-abdb-21985c9e88be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "db.add_documents(docs, batch_size=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "67210012-9765-4f41-a720-51f9c59880a6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27/27 [02:50<00:00,  6.31s/it]\n",
      "100%|██████████| 27/27 [02:36<00:00,  5.81s/it]\n",
      "100%|██████████| 27/27 [50:13<00:00, 111.59s/it]\n",
      "100%|██████████| 27/27 [15:32<00:00, 34.52s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import transformers\n",
    "\n",
    "transformers.logging.disable_progress_bar()\n",
    "transformers.logging.set_verbosity_error()\n",
    "\n",
    "models = [\n",
    "    \"HuggingFaceH4/zephyr-7b-beta\",\n",
    "    \"mistralai/Mistral-7B-Instruct-v0.2\",\n",
    "    \"HuggingFaceH4/zephyr-7b-gemma-v0.1\",\n",
    "    \"google/gemma-2b-it\",\n",
    "    #\"mistralai/Mixtral-8x7B-Instruct-v0.1\" GPU Memory\n",
    "]\n",
    "\n",
    "result_rows = []\n",
    "\n",
    "for model_id in models:\n",
    "    causal_lm = DecoderModel(model_id, device=\"cuda\")\n",
    "    with tqdm(total=len(questions)) as pbar:\n",
    "        for question in questions:\n",
    "            results = db.search(question, top_n=3)\n",
    "            contexts = [result[\"document\"] for result in results]\n",
    "            model_input = \"\\n\\n\".join(contexts)\n",
    "            answer = causal_lm(question, model_input)\n",
    "            result_rows.append(\n",
    "                {\n",
    "                    \"model\": model_id,\n",
    "                    \"contexts\": contexts,\n",
    "                    \"answer\": answer,\n",
    "                    \"question\": question\n",
    "                }\n",
    "            )\n",
    "            pbar.update(1)\n",
    "pd.DataFrame(result_rows).to_csv(f\"ML_BOOK_RESULTS.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7986d583-9fa8-4e02-a830-636f7041dff8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}