Completed Should Have Features

.github/workflows/node-js-tests.yml

@@ -31,10 +31,16 @@ jobs:
         cache-dependency-path: ./frontend/package-lock.json
     - name: Install Dependencies
       working-directory: ./frontend
-      run: npm ci
-    - name: Build Package
+      run: npm install
+    - name: Disable Next.js Telemetry
+      working-directory: ./frontend
+      run: npx next telemetry disable
+    - name: Test Build Package
       working-directory: ./frontend
       run: npm run build --if-present
+      env:
+        SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+        SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
   # - name: Run Test
     # working-directory: ./frontend
     # run: npm test

README.md

@@ -4,7 +4,7 @@ emoji: 📝
 colorFrom: blue
 colorTo: indigo
 sdk: docker
-python_version: 3.11.4
+python_version: 3.11.8
 app_port: 8000
 pinned: false
 ---
@@ -74,7 +74,7 @@ For more information, see the [DEPLOYMENT.md](./DEPLOYMENT.md).
 - [Python](https://python.org/) - Backend Server Environment
 - [FastAPI](https://fastapi.tiangolo.com/) - Backend API Web Framework
 - [LlamaIndex](https://www.llamaindex.ai/) - Data Framework for LLM
-- [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama) - LlamaIndex Application Bootstrap Tool
+- [create-llama](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama) - LlamaIndex Application Bootstrap Tool
 
 ## 📑 Contributing <a name = "contributing"></a>
 

backend/.gitignore

@@ -1,3 +1,10 @@
+# Production Files
 __pycache__
 storage
-.env
+.env
+**/data/EIR/
+
+# Dev Folders
+test.py
+storage backup
+data backup

backend/backend/app/api/routers/chat.py

@@ -4,17 +4,17 @@
 from fastapi import APIRouter, Depends, HTTPException, Request, status
 from fastapi.responses import StreamingResponse
 from fastapi.websockets import WebSocketDisconnect
-from llama_index import VectorStoreIndex
 from llama_index.llms.base import ChatMessage
 from llama_index.llms.types import MessageRole
 from llama_index.memory import ChatMemoryBuffer
 from llama_index.prompts import PromptTemplate
 from pydantic import BaseModel
 
+from backend.app.utils import auth
 from backend.app.utils.index import get_index
 from backend.app.utils.json import json_to_model
 
-chat_router = r = APIRouter()
+chat_router = r = APIRouter(dependencies=[Depends(auth.validate_user)])
 
 """
 This router is for chatbot functionality which consist of chat memory and chat engine.
@@ -34,6 +34,7 @@ class _Message(BaseModel):
 
 class _ChatData(BaseModel):
     messages: List[_Message]
+    document: str
 
 
 # custom prompt template to be used by chat engine
@@ -69,8 +70,13 @@ async def chat(
     # Note: To support clients sending a JSON object using content-type "text/plain",
     # we need to use Depends(json_to_model(_ChatData)) here
     data: _ChatData = Depends(json_to_model(_ChatData)),
-    index: VectorStoreIndex = Depends(get_index),
 ):
+    logger = logging.getLogger("uvicorn")
+    # get the document set selected from the request body
+    document_set = data.document
+    logger.info(f"Document Set: {document_set}")
+    # get the index for the selected document set
+    index = get_index(collection_name=document_set)
     # check preconditions and get last message
     if len(data.messages) == 0:
         raise HTTPException(
@@ -92,8 +98,6 @@ async def chat(
         for m in data.messages
     ]
 
-    logger = logging.getLogger("uvicorn")
-
     # query_engine = index.as_query_engine()
     # chat_engine = CondenseQuestionChatEngine.from_defaults(
     #     query_engine=query_engine,
@@ -102,9 +106,11 @@ async def chat(
     #     verbose=True,
     # )
 
+    logger.info(f"Messages: {messages}")
+
     memory = ChatMemoryBuffer.from_defaults(
         chat_history=messages,
-        token_limit=2000,
+        token_limit=3900,
     )
 
     logger.info(f"Memory: {memory.get()}")
@@ -114,17 +120,23 @@ async def chat(
         chat_mode="condense_plus_context",
         memory=memory,
         context_prompt=(
-            "You are a chatbot, able to have normal interactions, as well as talk"
-            " about information from documents regarding Public Sector Standard Conditions Of Contract (PSSCOC)."
+            "You are a helpful chatbot, able to have normal interactions, as well as answer questions"
+            " regarding information relating to the Public Sector Standard Conditions Of Contract (PSSCOC) Documents and JTC's Employer Information Requirements (EIR) Documents.\n"
+            "All the documents are in the context of the construction industry in Singapore.\n"
             "Here are the relevant documents for the context:\n"
             "{context_str}"
-            "\nInstruction: Based on the above documents, provide a detailed answer for the user question below. If you cannot answer the question, inform the user that you do not know."
+            "\nInstruction: Based on the above documents, provide a detailed answer for the user question below.\n"
+            "If you cannot answer the question or are unsure of how to answer, inform the user that you do not know.\n"
+            "If you need to clarify the question, ask the user for clarification.\n"
+            "You are to provide the relevant sources of which you got the information from in the context in brackets."
         ),
     )
     response = chat_engine.stream_chat(
         message=lastMessage.content, chat_history=messages
     )
 
+    # logger.info(f"Response Sources: {response.source_nodes}")
+
     # stream response
     async def event_generator():
         try:

backend/backend/app/api/routers/healthcheck.py

@@ -1,6 +1,12 @@
-from fastapi import APIRouter, Request
+import logging
+import os
 
-healthcheck_router = r = APIRouter()
+from fastapi import APIRouter, Depends, Request
+from supabase import Client, ClientOptions, create_client
+
+from backend.app.utils import auth
+
+healthcheck_router = r = APIRouter(dependencies=[Depends(auth.validate_user)])
 
 """
 This router is for healthcheck functionality.
@@ -19,14 +25,37 @@ async def healthcheck(
     # else:
     #     results["index"] = False
 
+    logger = logging.getLogger("uvicorn")
+
     # TODO: check if other services are ready
 
-    # logger.info("Healthcheck: {results}")
+    # Try to connect to supabase
+    supabase_url: str = os.environ.get("SUPABASE_URL")
+    supabase_key: str = os.environ.get("SUPABASE_ANON_KEY")
+
+    supabase: Client = create_client(
+        supabase_url=supabase_url,
+        supabase_key=supabase_key,
+        options=ClientOptions(
+            postgrest_client_timeout=10,
+            storage_client_timeout=10,
+        ),
+    )
+
+    response = supabase.table("users").select("id", count="exact").execute()
+
+    # logger.info(f"Supabase: {response}")
+
+    if response.count is not None:
+        results["supabase"] = True
+    else:
+        results["supabase"] = False
 
-    results = {"status": "OK"}
+    results["backend"] = True
+    logger.debug(f"Healthcheck: {results}")
     return results
 
 
 # Simple test to check if the healthcheck endpoint is working
 def test_healthcheck():
-    assert healthcheck() == {"status": "OK"}
+    assert healthcheck() == {"status": True, "supabase": True}

backend/backend/app/api/routers/query.py

@@ -8,10 +8,11 @@
 from llama_index.llms.types import MessageRole
 from pydantic import BaseModel
 
+from backend.app.utils import auth
 from backend.app.utils.index import get_index
 from backend.app.utils.json import json_to_model
 
-query_router = r = APIRouter()
+query_router = r = APIRouter(dependencies=[Depends(auth.validate_user)])
 
 """
 This router is for query functionality which consist of query engine.

backend/backend/app/api/routers/search.py

@@ -6,9 +6,10 @@
 from llama_index.postprocessor import SimilarityPostprocessor
 from llama_index.retrievers import VectorIndexRetriever
 
+from backend.app.utils import auth
 from backend.app.utils.index import get_index
 
-search_router = r = APIRouter()
+search_router = r = APIRouter(dependencies=[Depends(auth.validate_user)])
 
 """
 This router is for search functionality which consist of query engine.
@@ -22,8 +23,9 @@
 async def search(
     request: Request,
     index: VectorStoreIndex = Depends(get_index),
+    query: str = None,
 ):
-    query = request.query_params.get("query")
+    # query = request.query_params.get("query")
     logger = logging.getLogger("uvicorn")
     logger.info(f"Search: {query}")
     if query is None:

backend/backend/app/utils/auth.py

@@ -0,0 +1,141 @@
+import logging
+import os
+import time
+from typing import Dict
+
+import jwt
+from dotenv import load_dotenv
+from fastapi import HTTPException, Security
+from fastapi.security import APIKeyHeader
+from supabase import Client, ClientOptions, create_client
+
+load_dotenv()
+
+# Retrieve the API key header name from the environment
+API_AUTH_HEADER_NAME: str = os.getenv(
+    key="API_AUTH_HEADER_NAME", default="Authorization"
+)
+
+# Retrieve the Backend API key header from the environment
+API_KEY_HEADER_NAME: str = os.getenv(key="API_KEY_HEADER_NAME", default="X-API-Key")
+
+# Create an API key header instance
+API_AUTH_HEADER = APIKeyHeader(name=API_AUTH_HEADER_NAME, auto_error=False)
+
+# Retrieve the API key from the environment
+BACKEND_API_KEY: str = os.getenv(key="BACKEND_API_KEY")
+
+# Create an API key header instance
+API_KEY_HEADER = APIKeyHeader(name=API_KEY_HEADER_NAME, auto_error=False)
+
+JWT_SECRET: str = os.getenv("SUPABASE_JWT_SECRET")
+JWT_ALGORITHM = "HS256"
+
+
+def verify_jwt(jwtoken: str) -> bool:
+    """Verify the JWT token and return True if the token is valid, else return False"""
+    isTokenValid: bool = False
+
+    try:
+        payload = decodeJWT(jwtoken)
+    except Exception:
+        payload = None
+    if payload:
+        isTokenValid = True
+    return isTokenValid
+
+
+def decodeJWT(token: str) -> Dict:
+    """Decode the JWT token and return the payload if the token is valid, else return None"""
+    try:
+        decoded_token = jwt.decode(
+            token, JWT_SECRET, algorithms=[JWT_ALGORITHM], options={"verify_aud": False}
+        )
+        return decoded_token if decoded_token["exp"] >= time.time() else None
+    except Exception:
+        return None
+
+
+def get_user_from_JWT(token: str):
+    """Get the user id from the JWT token and return True if the user exists, else return False"""
+    supabase_url: str = os.environ.get("SUPABASE_URL")
+    supabase_key: str = os.environ.get("SUPABASE_ANON_KEY")
+
+    supabase: Client = create_client(
+        supabase_url=supabase_url,
+        supabase_key=supabase_key,
+        options=ClientOptions(
+            postgrest_client_timeout=10,
+            storage_client_timeout=10,
+        ),
+    )
+
+    payload = decodeJWT(token)
+    user_id = payload["sub"]
+
+    if user_id is not None:
+        # Try to get the user from the database using the user_id
+        response = supabase.table("users").select("*").eq("id", user_id).execute()
+        # print(response.data)
+        if len(response.data) == 0:
+            return False
+        return True
+    return False
+
+
+async def validate_user(
+    auth_token: str = Security(dependency=API_AUTH_HEADER),
+    api_key: str = Security(dependency=API_KEY_HEADER),
+):
+    try:
+        logger = logging.getLogger("uvicorn")
+        # logger.debug(f"Auth Token: {auth_token} | API Key: {api_key}")
+        if auth_token is not None or api_key is not None:
+            # If the access token is empty, use the 'X-API-Key' from the header
+            if auth_token is None:
+                # Access the 'X-API-Key' header directly
+                if BACKEND_API_KEY is None:
+                    raise ValueError("Backend API key is not set in Backend Service!")
+                # If the 'X-API-Key' does not match the backend API key, raise an error
+                if api_key != BACKEND_API_KEY:
+                    raise ValueError(
+                        "Invalid API key provided in the 'X-API-Key' header!"
+                    )
+                else:
+                    logger.info("Validated API key successfully!")
+                    return None
+            else:
+                auth_token = (
+                    auth_token.strip()
+                )  # Remove leading and trailing whitespaces
+                isBearer = auth_token.startswith(
+                    "Bearer"
+                )  # Check if the token starts with 'Bearer'
+                jwtoken = auth_token.split("Bearer ")[
+                    1
+                ]  # Extract the token from the 'Bearer' string
+                if JWT_SECRET is None:
+                    raise ValueError(
+                        "Supabase JWT Secret is not set in Backend Service!"
+                    )
+                if not isBearer:
+                    return (
+                        "Invalid token scheme. Please use the format 'Bearer [token]'"
+                    )
+                # Verify the JWT token is valid
+                if verify_jwt(jwtoken=jwtoken) is None:
+                    return "Invalid token. Please provide a valid token."
+                # Check if the user exists in the database
+                if get_user_from_JWT(token=jwtoken):
+                    logger.info("Validated User's Auth Token successfully!")
+                    return None
+                else:
+                    raise ValueError("User does not exist in the database!")
+        else:
+            raise ValueError(
+                "Either Access token [Authorization] or API key [X-API-Key] needed!"
+            )
+    except Exception as e:
+        logger = logging.getLogger("uvicorn")
+        logger.error(f"Error validating Auth Token / API key: {e}")
+        raise HTTPException(status_code=401, detail=f"Unauthorized - {e}")

backend/backend/app/utils/contants.py

@@ -30,6 +30,11 @@
 # Embedding Model Constants
 EMBED_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
 EMBED_POOLING = "mean"
+EMBED_MODEL_DIMENSIONS = 384  # MiniLM-L6-v2 uses 384 dimensions
+DEF_EMBED_MODEL_DIMENSIONS = (
+    1536  # Default embedding model dimensions used by OpenAI text-embedding-ada-002
+)
+EMBED_BATCH_SIZE = 100  # batch size for openai embeddings
 
 # Prompt Helper Constants
 # set maximum input size