4_process_front_end.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import all the necessary modules and global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from docai_module.config import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from docai_module.docai_methods import *\n",
    "from docai_module.storage_methods import *\n",
    "from docai_module.firestore_methods import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Architecture 1: API Centric"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Quick overview"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"./images/4_arch_front.png\"\n",
    "     alt=\"Processor\"\n",
    "     style=\"width:90%\"\n",
    "     />\n",
    "     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lab Details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cloud Storage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cloud Storage Methods\n",
    "\n",
    "In our architecture, everytime a system or a user needs to upload or download an object to the \"Original Document\" bucket, in this case a PDF document, it is necessary to generate a temporary signed URL to perform that activity (red dotted line).\n",
    "\n",
    "<img src=\"./images/4_11_storage_upload.png\"\n",
    "     style=\"width:40%\"\n",
    "     />\n",
    "\n",
    "Let's define a function to generate a signed url for download/upload and another function to download and upload the bytes itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generates a signed URL for downloading a blob using HTTP GET.\n",
    "def create_signed_url_download(\n",
    "    blob_name: str,\n",
    "    bucket_name: str\n",
    ") -> str:\n",
    "    bucket = STORAGE_CLIENT.bucket(bucket_name)\n",
    "    blob = bucket.blob(blob_name)\n",
    "\n",
    "    url = blob.generate_signed_url(\n",
    "        version=\"v4\",\n",
    "        expiration=datetime.timedelta(minutes=15),\n",
    "        method=\"GET\",\n",
    "    )\n",
    "    return url\n",
    "\n",
    "# Generates a signed URL for uploading a blob using HTTP POST.\n",
    "def create_signed_url_upload(\n",
    "    blob_name: str,\n",
    "    bucket_name: str,\n",
    "    content_type: str\n",
    "):\n",
    "    bucket = STORAGE_CLIENT.bucket(bucket_name)\n",
    "    blob = bucket.blob(blob_name)\n",
    "\n",
    "    signed_url = blob.generate_signed_url(\n",
    "        version=\"v4\",\n",
    "        expiration=datetime.timedelta(minutes=15),\n",
    "        method=\"PUT\",\n",
    "        content_type=content_type,\n",
    "    )\n",
    "    return signed_url\n",
    "\n",
    "# Download object from cloud storage\n",
    "def download_blob_bytes(bucket_name, source_blob_name):\n",
    "    \"\"\"Downloads a blob from the bucket.\"\"\"    \n",
    "    bucket = STORAGE_CLIENT.bucket(bucket_name)\n",
    "    blob = bucket.blob(source_blob_name)\n",
    "    return blob.download_as_bytes()\n",
    "\n",
    "# Upload bytes to cloud storage\n",
    "def upload_blob_bytes(bucket_name, \n",
    "                        image_bytes, \n",
    "                        destination_blob_name,\n",
    "                        content_type):\n",
    "    \"\"\"Uploads a file to the bucket.\"\"\"\n",
    "    bucket = STORAGE_CLIENT.bucket(bucket_name)\n",
    "    blob = bucket.blob(destination_blob_name)\n",
    "    blob.upload_from_string(image_bytes, content_type=content_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's do a quick test and generate a signed URL and upload a PDF to the ORIGINAL bucket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "signed_test_upload = create_signed_url_upload('loan_form.pdf', ORIGINAL_BUCKET, MIME_TYPE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://storage.googleapis.com/cool-ml-demos-original/loan_form.pdf?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=sa-geral%40cool-ml-demos.iam.gserviceaccount.com%2F20210129%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20210129T210617Z&X-Goog-Expires=900&X-Goog-SignedHeaders=content-type%3Bhost&X-Goog-Signature=07096e1f0f34675085aa01e0bfc71d8ac91c0f98af09c4cb00063cbfe24dac67c5793371c3f022c0a3cac0b15e05a4885af10e3904bb9e22dd8296d1504303e49871ad485312575e9fc9abb72726d62a51c5d2c15707f0e422afaf2ab1949131a74981caa8044f33975a9aed6bb29972d08bbc7f162f6c59d74d834627e94f7822f04f4230abda7e75cf0325669959b016592789360057151a4a90566d73afe5ca6236b3ef21c4a7a1652d1e74c8c13cf9c6eef65efe447f10f7ab72edc93d2ff2c613de8866b688598748b69016dd4f3528d86ad3a0e8fea02efdbd5e6f825f6cf811f3278ece64fdbeeb36a4da0f34f48a41a97ae50a786057ba1d52f4b040\n"
     ]
    }
   ],
   "source": [
    "print(signed_test_upload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "# Upload a file to this signed URL\n",
    "headers = {'Content-Type': 'application/pdf'}\n",
    "data = open('./files/loan_form.pdf', 'rb')\n",
    "r = requests.put(signed_test_upload, data=data, headers=headers)\n",
    "print(r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On the Google Cloud web console, navigate back to your bucket ORIGINAL-BUCKET and check if the object was uploaded properly.\n",
    "\n",
    "<img src=\"./images/4_7_storage_created.png\"\n",
    "     style=\"width:40%\"\n",
    "     />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Document AI\n",
    "\n",
    "We will use the processor created in the previous laboratory to call the API.  \n",
    "Recall that the type of the processor is \"Form Parsing\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ff4bad3352769404\n"
     ]
    }
   ],
   "source": [
    "print(PROCESSOR_ID)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use async processing with the method \"async_process_document\" defined in the Document AI lab."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the method to check if everything is working OK.  \n",
    "Let's use the same document we uploaded in our Cloud Storage test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "gcs_output_uri = 'gs://' + DOCAI_BUCKET\n",
    "gcs_output_uri_prefix = 'results'\n",
    "gcs_input_uri = 'gs://' + ORIGINAL_BUCKET + '/' + 'loan_form.pdf'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Operation ID: 12739865725164768158 created.\n",
      "Still processing, please wait ...\n",
      "Still processing, please wait ...\n",
      "Still processing, please wait ...\n",
      "Still processing, please wait ...\n",
      "Still processing, please wait ...\n",
      "Still processing, please wait ...\n",
      "Still processing, please wait ...\n",
      "Still processing, please wait ...\n",
      "Finish processing the document.\n"
     ]
    }
   ],
   "source": [
    "OP_ID = async_process_document(gcs_input_uri, gcs_output_uri, gcs_output_uri_prefix, MIME_TYPE)\n",
    "op_id_only = OP_ID.operation.name.split(sep='/')[-1]\n",
    "\n",
    "print(f'Operation ID: {op_id_only} created.')\n",
    "import time\n",
    "while not OP_ID.done():\n",
    "    print('Still processing, please wait ...')\n",
    "    time.sleep(10)\n",
    "else:\n",
    "    print('Finish processing the document.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On the Google Cloud web console, navigate back to your bucket and check if there is a folder \"results\".  \n",
    "This folder will hold the results from the Document AI call.\n",
    "\n",
    "<img src=\"./images/4_8_results_folder.png\"\n",
    "     style=\"width:50%\"\n",
    "     />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loss Prevention\n",
    "\n",
    "The next step in our architecture is to create a method to anonymize sensitive information from our document.  \n",
    "We will use a simple function to identify any street address present in the JSON file.\n",
    "\n",
    "For the sake of simplicity, we will inspect the document with a basic DLP API call.\n",
    "\n",
    "<img src=\"./images/4_3_arch_dlp.png\"\n",
    "     style=\"width:70%\"\n",
    "     />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Method to detect street address using DLP API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inspect_string(content_string):\n",
    "    # Prepare info_types by converting the list of strings into a list of\n",
    "    # dictionaries (protos are also accepted).\n",
    "    # The info types to search for in the content. Required.\n",
    "    info_types = [{\"name\": \"STREET_ADDRESS\"}]\n",
    "\n",
    "    # The minimum likelihood to constitute a match. Optional.\n",
    "    min_likelihood = dlp_v2.Likelihood.POSSIBLE\n",
    "\n",
    "    # The maximum number of findings to report (0 = server maximum). Optional.\n",
    "    max_findings = 0\n",
    "\n",
    "    # Whether to include the matching string in the results. Optional.\n",
    "    include_quote = True\n",
    "\n",
    "    # Construct the configuration dictionary. Keys which are None may\n",
    "    # optionally be omitted entirely.\n",
    "    inspect_config = {\n",
    "        \"info_types\": info_types,\n",
    "        \"min_likelihood\": min_likelihood,\n",
    "        \"include_quote\": include_quote,\n",
    "        \"limits\": {\"max_findings_per_request\": max_findings},\n",
    "    }\n",
    "\n",
    "    # Construct the `item`.\n",
    "    item = {\"value\": content_string}\n",
    "\n",
    "    # Convert the project id into a full resource id.\n",
    "    parent = f\"projects/{PROJECT_ID}\"\n",
    "\n",
    "    # Call the API.\n",
    "    response = DLP_CLIENT.inspect_content(\n",
    "        request={\"parent\": parent, \"inspect_config\": inspect_config, \"item\": item}\n",
    "    )\n",
    "\n",
    "    # Print out the results.\n",
    "    if response.result.findings:\n",
    "        for finding in response.result.findings:\n",
    "            try:\n",
    "                if finding.quote:\n",
    "                    print(\"Quote: {}\".format(finding.quote))\n",
    "            except AttributeError:\n",
    "                pass\n",
    "            print(\"Info type: {}\".format(finding.info_type.name))\n",
    "            print(\"Likelihood: {}\".format(finding.likelihood))\n",
    "    else:\n",
    "        print(\"No findings.\")\n",
    "\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Loan Agreement Form\\nAgreement Number:\\n0123456789\\nAgreement date:\\n01/01/2020\\nThis loan agreement is commenced between the parties:\\nMortgage company contact details:\\nName:\\nMortgage company A\\nAddress:\\n100 Franklin Street, Mountain View, CA, 94035\\nPhone number: 1-800-843-8623\\n(hereinafter referred to as the lender)\\nIndividual details:\\nName:\\nArjun Patel\\nMarital status:\\nSingle\\nMarried ☐\\nOther\\nAddress:\\n500 Castro Street, Mountain View, CA 94035\\nPhone number: 650-987-0934\\n(hereinafter referred to as the borrower)\\n[Fill in all details as per instructions]\\n6.0\\n%.\\nThe lender is ready to sanction $ 2000 as the loan amount at\\n[Total loan amount along with the agreed percentage rate].\\nThis loan agreement is valid from 01/01/2020 and is ending on 12/31/2020.\\nTerms & agreements:\\n38.67\\nper month for\\n5\\nyears.\\nThe borrower will pay an installment of $\\n[Amount & tenure of loan]\\nAny late installment will be accepted with $\\n40\\nas a fine.\\n'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "with open('./files/result_loan_form.json') as f:\n",
    "    data = json.load(f)\n",
    "data['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quote: 100 Franklin Street, Mountain View, CA, 94035\n",
      "Info type: STREET_ADDRESS\n",
      "Likelihood: 4\n",
      "Quote: 500 Castro Street, Mountain View, CA 94035\n",
      "Info type: STREET_ADDRESS\n",
      "Likelihood: 4\n"
     ]
    }
   ],
   "source": [
    "dlp_response = inspect_string(data['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Firestore\n",
    "\n",
    "In this architecture we will use Firestore database to store the results of the API calls.  \n",
    "After each API call to Document AI, DLP, etc., the location of the result or the intermediate ID of the Long Running Operation (LRO) will be stored there.  \n",
    "\n",
    "<img src=\"./images/4_4_arch_firestore.png\"\n",
    "     style=\"width:70%\"\n",
    "     />\n",
    "     \n",
    "     \n",
    "Next, let's define some methods to interact with Firestore."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-42-33f5d23f62fc>, line 6)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-42-33f5d23f62fc>\"\u001b[0;36m, line \u001b[0;32m6\u001b[0m\n\u001b[0;31m    'name':'blob_name'\u001b[0m\n\u001b[0m          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "parent = f'projects/{PROJECT_ID}/databases/(default)/collectionGroups/documents'\n",
    "index = {\n",
    "    'name':'documents',\n",
    "    'query_scope': 'COLLECTION',\n",
    "    'fields': {\n",
    "        'blob_name'}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Parameter to MergeFrom() must be instance of same class: expected google.firestore.admin.v1.Index.IndexField got str.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-5f0ace937f88>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mFIRESTORE_ADMIN\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/envs/ml-demos3.8/lib/python3.8/site-packages/google/cloud/firestore_admin_v1/services/firestore_admin/client.py\u001b[0m in \u001b[0;36mcreate_index\u001b[0;34m(self, request, parent, index, retry, timeout, metadata)\u001b[0m\n\u001b[1;32m    459\u001b[0m                 \u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 461\u001b[0;31m                 \u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    462\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    463\u001b[0m         \u001b[0;31m# Wrap the RPC method; this adds retry and timeout information,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/ml-demos3.8/lib/python3.8/site-packages/proto/message.py\u001b[0m in \u001b[0;36m__setattr__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m    582\u001b[0m         \u001b[0mmarshal\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_meta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmarshal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    583\u001b[0m         \u001b[0mpb_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_meta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpb_type\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 584\u001b[0;31m         \u001b[0mpb_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmarshal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_proto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpb_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    586\u001b[0m         \u001b[0;31m# Clear the existing field.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/ml-demos3.8/lib/python3.8/site-packages/proto/marshal/marshal.py\u001b[0m in \u001b[0;36mto_proto\u001b[0;34m(self, proto_type, value, strict)\u001b[0m\n\u001b[1;32m    203\u001b[0m         \u001b[0;31m# Convert ordinary values.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    204\u001b[0m         \u001b[0mrule\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_rules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproto_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_noop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 205\u001b[0;31m         \u001b[0mpb_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_proto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m         \u001b[0;31m# Sanity check: If we are in strict mode, did we get the value we want?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/ml-demos3.8/lib/python3.8/site-packages/proto/marshal/rules/message.py\u001b[0m in \u001b[0;36mto_proto\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m     30\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_map\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_descriptor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Parameter to MergeFrom() must be instance of same class: expected google.firestore.admin.v1.Index.IndexField got str."
     ]
    }
   ],
   "source": [
    "FIRESTORE_ADMIN.create_index(parent=parent, index=index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a document in the collection with doc_values\n",
    "def create_document_firestore(\n",
    "    collection: str,\n",
    "    doc_values: dict\n",
    "):\n",
    "    FIRESTORE_CLIENT.collection(collection).add(doc_values)\n",
    "\n",
    "def create_firestore_index(\n",
    "    collection_id: str,    \n",
    "):\n",
    "    parent = f'projects/{PROJECT_ID}/databases/(default)/collectionGroups/{collection_id}'\n",
    "    index = {\n",
    "        'name':'documents',\n",
    "        'fields': [\n",
    "            {'name':'blob_name'},{'name':'bucket_name'} \n",
    "        ]\n",
    "    }\n",
    "    return FIRESTORE_ADMIN.create_index(parent, index)\n",
    "    \n",
    "# Query all documents from collection, with filter\n",
    "def get_all_documents_firebase(\n",
    "    collection: str,\n",
    "    field_name: str,\n",
    "    filter_value: str,\n",
    "    limit: int = 1\n",
    ") -> dict:\n",
    "    docs = FIRESTORE_CLIENT.collection(collection).where(\n",
    "        field_name, u'==', filter_value).order_by(\n",
    "            'creation_time').limit(limit).stream()\n",
    "    response = {}\n",
    "\n",
    "    for doc in docs:\n",
    "        response[doc.id] = doc.to_dict()\n",
    "\n",
    "    return response\n",
    "\n",
    "# Get document content with blob_name\n",
    "def get_document_firebase(\n",
    "    collection: str,\n",
    "    blob_name: str\n",
    ") -> dict:\n",
    "    doc = FIRESTORE_CLIENT.collection(collection).where(\n",
    "            'blob_name', '==', blob_name).limit(1).get()\n",
    "    \n",
    "    return doc[0].to_dict() if doc else doc\n",
    "\n",
    "# Get specific field from document in collection\n",
    "def get_field_from_doc_firebase(\n",
    "    collection: str,\n",
    "    blob_name: str,\n",
    "    field_name: str\n",
    ") -> dict:\n",
    "    doc_field = FIRESTORE_CLIENT.collection(collection).where(\n",
    "            'blob_name', '==', blob_name).limit(1).get()\n",
    "\n",
    "    return doc_field[0].get(field_name) if doc_field else doc_field\n",
    "\n",
    "# Update a specific document field\n",
    "def update_document_field_firebase(\n",
    "    collection: str,\n",
    "    blob_name: str,\n",
    "    field_name: str,\n",
    "    value: str\n",
    ") -> bool:\n",
    "    doc_id = FIRESTORE_CLIENT.collection(collection).where(\n",
    "        'blob_name', '==', blob_name).limit(1).get()\n",
    "\n",
    "    if doc_id:\n",
    "        doc_ref = FIRESTORE_CLIENT.collection(\n",
    "            'documents').document(doc_id[0].id)\n",
    "        doc_ref.update({field_name:value})\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's test and understand how each component works."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The following function call will create a document in collection 'document' with doc_values\n",
    "doc_values = {'user_id': 'user','creation_time':'now',\n",
    "                'blob_name':'loan_form.pdf','bucket_name':ORIGINAL_BUCKET, 'content_type':'application/pdf'}\n",
    "create_document_firestore('documents', doc_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To verify if the document was created, navigate to Cloud Firestore in the web console.\n",
    "\n",
    "<img src=\"./images/4_9_firestore.png\"\n",
    "     style=\"width:20%\"\n",
    "     />\n",
    "     \n",
    "Then, click on the document we just created and check if the information is there:\n",
    "\n",
    "<img src=\"./images/4_10_firestore_info.png\"\n",
    "     style=\"width:70%\"\n",
    "     />\n",
    "     \n",
    "As we specified just the collection_id as \"documents\", the document_id was assigned with a random number."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, let's create an index in Firebase to allow more complex queries.\n",
    "\n",
    "To keep it simple, we will index both blob_name and bucket_name from documents fields."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "create_index() takes from 1 to 2 positional arguments but 3 were given",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-30-74cc3a986fbe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcreate_firestore_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'documents'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-26-d70920786fde>\u001b[0m in \u001b[0;36mcreate_firestore_index\u001b[0;34m(collection_id)\u001b[0m\n\u001b[1;32m     16\u001b[0m         ]\n\u001b[1;32m     17\u001b[0m     }\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mFIRESTORE_ADMIN\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;31m# Query all documents from collection, with filter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: create_index() takes from 1 to 2 positional arguments but 3 were given"
     ]
    }
   ],
   "source": [
    "create_firestore_index('documents')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_get_all_documents_firebase('documents', 'blob_name', 'loan_form.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_get_document_firebase('documents', 'loan_form.pdf')"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "name": "tf2-gpu.2-4.mnightly-2021-01-20-debian-10-test",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-4:mnightly-2021-01-20-debian-10-test"
  },
  "kernelspec": {
   "display_name": "ml-demos3.8",
   "language": "python",
   "name": "ml-demos3.8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}