From d9b91edc7a1e931b56f791c008437adbc174af84 Mon Sep 17 00:00:00 2001 From: Yee Kit Date: Fri, 26 Jan 2024 22:49:14 +0800 Subject: [PATCH 1/3] Added timeout to fetch components --- frontend/app/components/ui/search/useSearch.tsx | 4 +++- frontend/app/status/page.tsx | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/ui/search/useSearch.tsx b/frontend/app/components/ui/search/useSearch.tsx index b30bebb..ad3bf46 100644 --- a/frontend/app/components/ui/search/useSearch.tsx +++ b/frontend/app/components/ui/search/useSearch.tsx @@ -32,7 +32,9 @@ const useSearch = (): UseSearchResult => { setIsLoading(false); return; } - const response = await fetch(`${search_api}?query=${query}`); + const response = await fetch(`${search_api}?query=${query}`, { + signal: AbortSignal.timeout(10000), // Abort the request if it takes longer than 10 seconds + }); const data = await response.json(); setSearchResults(data); } catch (error) { diff --git a/frontend/app/status/page.tsx b/frontend/app/status/page.tsx index f7c25d3..a9a2729 100644 --- a/frontend/app/status/page.tsx +++ b/frontend/app/status/page.tsx @@ -12,7 +12,9 @@ const StatusPage = () => { const { data, error, isValidating, mutate } = useSWR(healthcheck_api, async (url) => { try { // Fetch the data - const response = await fetch(url); + const response = await fetch(url, { + signal: AbortSignal.timeout(5000), // Abort the request if it takes longer than 5 seconds + }); if (!response.ok) { throw new Error(response.statusText || 'Unknown Error'); } From 52ec1f973f84efe4d4431d18bb720ccbed259787 Mon Sep 17 00:00:00 2001 From: Yee Kit Date: Fri, 26 Jan 2024 23:06:29 +0800 Subject: [PATCH 2/3] Update pyproject.toml with better install options & instructions for cpu/cuda --- backend/README.md | 54 +++++++++++++++++++++++++++++------------- backend/pyproject.toml | 23 ++++++++++-------- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/backend/README.md b/backend/README.md index b25a231..f4c32c3 100644 --- a/backend/README.md +++ b/backend/README.md @@ -5,34 +5,56 @@ The backend is built using Python & [FastAPI](https://fastapi.tiangolo.com/) boo ## Requirements 1. Python >= 3.11 -2. Poetry (To manage dependencies) +2. Miniconda (To manage Python versions) + - [Windows](https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe) + - [Linux](https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh) + - [MacOS](https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.pkg) + - ```conda create -n SmartRetrieval python=3.11``` +3. Pipx (To manage Python packages) + - ```pip install pipx``` (If you already have pipx installed, you can skip this step) +4. Cuda > 12.1 (if you have a Nvidia GPU) + - [Windows](https://developer.nvidia.com/cuda-downloads) + - [Linux](https://developer.nvidia.com/cuda-downloads) + - [MacOS](https://developer.nvidia.com/cuda-downloads) +5. Poetry (To manage dependencies) - ```pipx install poetry``` ## Getting Started -First, setup the `pyproject.toml` file to install the correct version of PyTorch (CPU or GPU): +First, ensure if you want to use the cuda version of pytorch, you have the correct version `cuda > 12.1` of cuda installed. You can check this by running `nvcc --version or nvidia-smi` in your terminal. If you do not have cuda installed, you can install it from [here](https://developer.nvidia.com/cuda-downloads). -Comment/Uncomment the following block depending on your system. Use CPU only if you do not have a supported Cuda Device. +Ensure you have followed the steps in the requirements section above. + +Then activate the conda environment: ```bash -# For CPU version: Windows and Linux and MacOS (arm64) -torch = [ - { url = "https://download.pytorch.org/whl/cpu/torch-2.1.1%2Bcpu-cp311-cp311-win_amd64.whl", markers = "sys_platform == 'win32'" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.1.1%2Bcpu-cp311-cp311-linux_x86_64.whl", markers = "sys_platform == 'linux'" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.1.1-cp311-none-macosx_11_0_arm64.whl", markers = "sys_platform == 'darwin'" }, -] -## For GPU version: Windows and Linux and MacOS (arm64) -# torch = [ -# { url = "https://download.pytorch.org/whl/cu121/torch-2.1.1%2Bcu121-cp311-cp311-win_amd64.whl", markers = "sys_platform == 'win32'" }, -# { url = "https://download.pytorch.org/whl/cu121/torch-2.1.1%2Bcu121-cp311-cp311-linux_x86_64.whl", markers = "sys_platform == 'linux'" }, -# { url = "https://download.pytorch.org/whl/cu121/torch-2.1.1-cp311-none-macosx_11_0_arm64.whl", markers = "sys_platform == 'darwin'" }, -# ] +conda activate SmartRetrieval ``` Second, setup the environment: ```bash -poetry install +# Only run one of the following commands: +----------------------------------------------- +# Install dependencies and torch (cpu version) +# Windows: Set env for llama-cpp-python with openblas support on cpu +$env:CMAKE_ARGS = "-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" +# Linux: Set env for llama-cpp-python with openblas support on cpu +CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" +# Then: +poetry install --without torch-cuda +----------------------------------------------- +# Install dependencies and torch (cuda version) +# Windows: Set env for llama-cpp-python with cuda support on gpu +$env:CMAKE_ARGS = "-DLLAMA_CUBLAS=on" +# Linux: Set env for llama-cpp-python with cuda support on gpu +CMAKE_ARGS="-DLLAMA_CUBLAS=on" +# Then: +poetry install --without torch-cpu +``` + +```bash +# Enter poetry shell poetry shell ``` diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 9ad587a..5f30940 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -15,21 +15,24 @@ pypdf = "^3.17.0" python-dotenv = "^1.0.0" llama-cpp-python = "^0.2.18" transformers = "^4.35.2" -# For CPU version: Windows and Linux and MacOS (arm64) -torch = [ - { url = "https://download.pytorch.org/whl/cpu/torch-2.1.1%2Bcpu-cp311-cp311-win_amd64.whl", markers = "sys_platform == 'win32'" }, - { url = "https://download.pytorch.org/whl/cpu/torch-2.1.1%2Bcpu-cp311-cp311-linux_x86_64.whl", markers = "sys_platform == 'linux'" }, -] -## For GPU version: Windows and Linux and MacOS (arm64) -# torch = [ -# { url = "https://download.pytorch.org/whl/cu121/torch-2.1.1%2Bcu121-cp311-cp311-win_amd64.whl", markers = "sys_platform == 'win32'" }, -# { url = "https://download.pytorch.org/whl/cu121/torch-2.1.1%2Bcu121-cp311-cp311-linux_x86_64.whl", markers = "sys_platform == 'linux'" }, -# ] docx2txt = "^0.8" # Dev Dependencies here [tool.poetry.group.dev.dependencies] +# For CPU torch version: Windows and Linux +[tool.poetry.group.torch-cpu.dependencies] +torch = [ + { url = "https://download.pytorch.org/whl/cpu/torch-2.1.1%2Bcpu-cp311-cp311-win_amd64.whl", markers = "sys_platform == 'win32'" }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.1.1%2Bcpu-cp311-cp311-linux_x86_64.whl", markers = "sys_platform == 'linux'" }, +] + +## For Cuda torch version: Windows and Linux +[tool.poetry.group.torch-cuda.dependencies] +torch = [ + { url = "https://download.pytorch.org/whl/cu121/torch-2.1.1%2Bcu121-cp311-cp311-win_amd64.whl", markers = "sys_platform == 'win32'" }, + { url = "https://download.pytorch.org/whl/cu121/torch-2.1.1%2Bcu121-cp311-cp311-linux_x86_64.whl", markers = "sys_platform == 'linux'" }, +] [build-system] requires = ["poetry-core"] From 48679e8b74936ad65c3800ac39c536a1997e946e Mon Sep 17 00:00:00 2001 From: Yee Kit Date: Fri, 26 Jan 2024 23:06:50 +0800 Subject: [PATCH 3/3] Update Dockerfile with Cuda support by default --- Dockerfile | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f318ce..bef2e7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,52 @@ -# Use the official Python Image -FROM python:3.11.4 +ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04" +# Build with Cuda 12.1.1 and Ubuntu 22.04 +FROM nvidia/cuda:${CUDA_IMAGE} # Set up a new user named "user" with user ID 1000 RUN useradd -m -u 1000 user + # Install the dependencies -RUN apt-get update && apt-get -y install cmake protobuf-compiler libopenblas-dev liblapack-dev liblapacke-dev libeigen3-dev libboost-all-dev +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y git build-essential \ + python3.11 gcc wget \ + ocl-icd-opencl-dev opencl-headers clinfo \ + cmake protobuf-compiler pkg-config \ + libclblast-dev libopenblas-dev \ + liblapack-dev liblapacke-dev libeigen3-dev libboost-all-dev \ + && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +# Install pip for python 3.11 +RUN wget https://bootstrap.pypa.io/get-pip.py && \ + python3.11 get-pip.py && \ + rm get-pip.py # Switch to the user 'user' USER user +# Setting build related env vars +ENV CUDA_DOCKER_ARCH=all +ENV LLAMA_CUBLAS=1 + # Set home to the user's home directory and Poetry's environment variables ENV HOME=/home/user \ PATH=/home/user/.local/bin:$PATH \ + PYTHONUNBUFFERED=1 \ POETRY_NO_INTERACTION=1 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ POETRY_VIRTUALENVS_CREATE=1 \ POETRY_CACHE_DIR=/tmp/poetry_cache \ - CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" + # Build llama-cpp-python with default cuda support + CMAKE_ARGS="-DLLAMA_CUBLAS=on" + # CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" # Set the working directory to /app WORKDIR $HOME/app # Update pip and wheel -RUN pip install --upgrade pip setuptools wheel +RUN python3.11 -m pip install --upgrade pip setuptools wheel # Install poetry -RUN pip install poetry +RUN python3.11 -m pip install poetry # Copy the poetry files COPY --chown=user ./backend/pyproject.toml ./backend/poetry.lock $HOME/app/ @@ -34,8 +55,7 @@ COPY --chown=user ./backend/pyproject.toml ./backend/poetry.lock $HOME/app/ COPY --chown=user ./backend $HOME/app # Install the dependencies -RUN poetry lock --no-update -RUN poetry install --without dev && \ +RUN poetry install --without dev,torch-cpu && \ rm -rf /tmp/poetry_cache # Change to the package directory