diff --git a/VideoRAGQnA/Dockerfile b/VideoQnA/Dockerfile similarity index 88% rename from VideoRAGQnA/Dockerfile rename to VideoQnA/Dockerfile index 7eadbfb8b..9712d3398 100644 --- a/VideoRAGQnA/Dockerfile +++ b/VideoQnA/Dockerfile @@ -22,7 +22,7 @@ WORKDIR /home/user/GenAIComps RUN pip install --no-cache-dir --upgrade pip && \ pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt -COPY ./videoragqna.py /home/user/videoragqna.py +COPY ./videoqna.py /home/user/videoqna.py ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps @@ -30,4 +30,4 @@ USER user WORKDIR /home/user -ENTRYPOINT ["python", "videoragqna.py"] +ENTRYPOINT ["python", "videoqna.py"] diff --git a/VideoRAGQnA/README.md b/VideoQnA/README.md similarity index 58% rename from VideoRAGQnA/README.md rename to VideoQnA/README.md index e3b9dded9..6f2e1d863 100644 --- a/VideoRAGQnA/README.md +++ b/VideoQnA/README.md @@ -1,8 +1,8 @@ -# VideoRAGQnA Application +# VideoQnA Application -Video RAG QnA is a framework that retrieves video based on provided user prompt. It uses only the video embeddings to perform vector similarity search in Intel's VDMS vector database and performs all operations on Intel Xeon CPU. The pipeline supports long form videos and time-based search. +VideoQnA is a framework that retrieves video based on provided user prompt. It uses only the video embeddings to perform vector similarity search in Intel's VDMS vector database and performs all operations on Intel Xeon CPU. The pipeline supports long form videos and time-based search. -VideoRAGQnA is implemented on top of [GenAIComps](https://github.com/opea-project/GenAIComps), with the architecture flow chart shows below: +VideoQnA is implemented on top of [GenAIComps](https://github.com/opea-project/GenAIComps), with the architecture flow chart shows below: ```mermaid --- @@ -21,9 +21,9 @@ flowchart LR classDef orange fill:#FBAA60,stroke:#ADD8E6,stroke-width:2px,fill-opacity:0.5 classDef orchid fill:#C26DBC,stroke:#ADD8E6,stroke-width:2px,fill-opacity:0.5 classDef invisible fill:transparent,stroke:transparent; - style VideoRAGQnA-MegaService stroke:#000000 + style VideoQnA-MegaService stroke:#000000 %% Subgraphs %% - subgraph VideoRAGQnA-MegaService["VideoRAGQnA-MegaService"] + subgraph VideoQnA-MegaService["VideoQnA-MegaService"] direction LR EM([Embedding
]):::blue RET([Retrieval
]):::blue @@ -36,10 +36,10 @@ flowchart LR UI([UI server
]):::orchid Ingest([Ingest
]):::orchid end - subgraph VideoRAGQnA GateWay + subgraph VideoQnA GateWay direction LR invisible1[ ]:::invisible - GW([VideoRAGQnA GateWay
]):::orange + GW([VideoQnA GateWay
]):::orange end subgraph . X([OPEA Micsrservice]):::blue @@ -66,7 +66,7 @@ flowchart LR direction LR a[User Input Query] -->|1| UI UI -->|2| GW - GW <==>|3| VideoRAGQnA-MegaService + GW <==>|3| VideoQnA-MegaService EM ==>|4| RET RET ==>|5| RER RER ==>|6| LLM @@ -85,11 +85,14 @@ flowchart LR DP <-.->|d|VDB ``` -This VideoRAGQnA use case performs RAG using LangChain, Intel VDMS VectorDB and Text Generation Inference on Intel XEON Scalable Processors. +- This project implements a Retrieval-Augmented Generation (RAG) workflow using LangChain, Intel VDMS VectorDB, and Text Generation Inference, optimized for Intel XEON Scalable Processors. +- Video Processing: Videos are converted into feature vectors using mean aggregation and stored in the VDMS vector store. +- Query Handling: When a user submits a query, the system performs a similarity search in the vector store to retrieve the best-matching videos. +- Contextual Inference: The retrieved videos are then sent to the Large Vision Model (LVM) for inference, providing supplemental context for the query. -## Deploy VideoRAGQnA Service +## Deploy VideoQnA Service -The VideoRAGQnA service can be effortlessly deployed on Intel XEON Scalable Processors. +The VideoQnA service can be effortlessly deployed on Intel XEON Scalable Processors. ### Required Models @@ -100,8 +103,8 @@ By default, the embedding and LVM models are set to a default value as listed be | Embedding | openai/clip-vit-base-patch32 | | LVM | DAMO-NLP-SG/Video-LLaMA | -### Deploy VideoRAGQnA on Xeon +### Deploy VideoQnA on Xeon For full instruction of deployment, please check [Guide](docker_compose/intel/cpu/xeon/README.md) -Currently we support deploying VideoRAGQnA services with docker compose, using the docker images `built from source`. Find the corresponding [compose.yaml](docker_compose/intel/cpu/xeon/compose.yaml). +Currently we support deploying VideoQnA services with docker compose, using the docker images `built from source`. Find the corresponding [compose.yaml](docker_compose/intel/cpu/xeon/compose.yaml). diff --git a/VideoRAGQnA/assets/img/video-rag-qna.gif b/VideoQnA/assets/img/videoqna.gif similarity index 100% rename from VideoRAGQnA/assets/img/video-rag-qna.gif rename to VideoQnA/assets/img/videoqna.gif diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md b/VideoQnA/docker_compose/intel/cpu/xeon/README.md similarity index 84% rename from VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md rename to VideoQnA/docker_compose/intel/cpu/xeon/README.md index 2c8426108..47af1b443 100644 --- a/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md +++ b/VideoQnA/docker_compose/intel/cpu/xeon/README.md @@ -1,8 +1,8 @@ -# Build Mega Service of videoragqna on Xeon +# Build Mega Service of videoqna on Xeon -This document outlines the deployment process for a videoragqna application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `embedding`, `retriever`, `rerank`, and `lvm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. +This document outlines the deployment process for a videoqna application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `embedding`, `retriever`, `rerank`, and `lvm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. -VideoRAGQnA is a pipeline that retrieves video based on provided user prompt. It uses only the video embeddings to perform vector similarity search in Intel's VDMS vector database and performs all operations on Intel Xeon CPU. The pipeline supports long form videos and time-based search. +VideoQnA is a pipeline that retrieves video based on provided user prompt. It uses only the video embeddings to perform vector similarity search in Intel's VDMS vector database and performs all operations on Intel Xeon CPU. The pipeline supports long form videos and time-based search. ## 🚀 Port used for the microservices @@ -68,7 +68,7 @@ docker build -t opea/retriever-vdms:latest --build-arg https_proxy=$https_proxy ### 3. Build Rerank Image ```bash -docker build -t opea/reranking-videoragqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/video-rag-qna/Dockerfile . +docker build -t opea/reranking-videoqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/videoqna/Dockerfile . ``` ### 4. Build LVM Image (Xeon) @@ -89,14 +89,14 @@ cd .. ### 6. Build MegaService Docker Image -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `videoragqna.py` Python script. +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `videoqna.py` Python script. Build MegaService Docker image via below command: ```bash git clone https://github.com/opea-project/GenAIExamples.git -cd GenAIExamples/VideoRAGQnA/ -docker build -t opea/videoragqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +cd GenAIExamples/VideoQnA/ +docker build -t opea/videoqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . ``` ### 7. Build UI Docker Image @@ -105,7 +105,7 @@ Build frontend Docker image via below command: ```bash cd ui -docker build -t opea/videoragqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +docker build -t opea/videoqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . ``` Then run the command `docker images`, you will have the following 8 Docker Images: @@ -113,11 +113,11 @@ Then run the command `docker images`, you will have the following 8 Docker Image 1. `opea/dataprep-multimodal-vdms:latest` 2. `opea/embedding-multimodal-clip:latest` 3. `opea/retriever-vdms:latest` -4. `opea/reranking-videoragqna:latest` +4. `opea/reranking-videoqna:latest` 5. `opea/video-llama-lvm-server:latest` 6. `opea/lvm-video-llama:latest` -7. `opea/videoragqna:latest` -8. `opea/videoragqna-ui:latest` +7. `opea/videoqna:latest` +8. `opea/videoqna-ui:latest` ## 🚀 Start Microservices @@ -168,7 +168,7 @@ export DATAPREP_GET_VIDEO_LIST_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_ export VDMS_HOST=${host_ip} export VDMS_PORT=8001 -export INDEX_NAME="mega-videoragqna" +export INDEX_NAME="mega-videoqna" export LLM_DOWNLOAD="True" export USECLIP=1 @@ -189,7 +189,7 @@ There are 2 parts of the pipeline: In the deploy steps, you need to start the VDMS DB and dataprep firstly, then insert some sample data into it. After that you could get the megaservice up. ```bash -cd GenAIExamples/VideoRAGQnA/docker_compose/intel/cpu/xeon/ +cd GenAIExamples/VideoQnA/docker_compose/intel/cpu/xeon/ docker volume create video-llama-model docker compose up vdms-vector-db dataprep -d @@ -306,7 +306,7 @@ docker compose up -d 7. MegaService ```bash - curl http://${host_ip}:8888/v1/videoragqna -H "Content-Type: application/json" -d '{ + curl http://${host_ip}:8888/v1/videoqna -H "Content-Type: application/json" -d '{ "messages": "What is the man doing?", "stream": "True" }' @@ -319,16 +319,16 @@ docker compose up -d To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below: ```yaml - videoragqna-xeon-ui-server: - image: opea/videoragqna-ui:latest + videoqna-xeon-ui-server: + image: opea/videoqna-ui:latest ... ports: - "80:5173" # port map to host port 80 ``` -Here is an example of running videoragqna: +Here is an example of running videoqna: -![project-screenshot](../../../../assets/img/video-rag-qna.gif) +![project-screenshot](../../../../assets/img/videoqna.gif) ## Clean Microservices diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml b/VideoQnA/docker_compose/intel/cpu/xeon/compose.yaml similarity index 89% rename from VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml rename to VideoQnA/docker_compose/intel/cpu/xeon/compose.yaml index 488be51ab..47479a414 100644 --- a/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/VideoQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -61,8 +61,8 @@ services: volumes: - /home/$USER/.cache/huggingface/hub:/home/user/.cache/huggingface/hub reranking: - image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest} - container_name: reranking-videoragqna-server + image: ${REGISTRY:-opea}/reranking-videoqna:${TAG:-latest} + container_name: reranking-videoqna-server ports: - "8000:8000" ipc: host @@ -103,9 +103,9 @@ services: restart: unless-stopped depends_on: - lvm-video-llama - videoragqna-xeon-backend-server: - image: ${REGISTRY:-opea}/videoragqna:${TAG:-latest} - container_name: videoragqna-xeon-backend-server + videoqna-xeon-backend-server: + image: ${REGISTRY:-opea}/videoqna:${TAG:-latest} + container_name: videoqna-xeon-backend-server depends_on: - vdms-vector-db - dataprep @@ -116,7 +116,7 @@ services: - lvm ports: - "8888:8888" - entrypoint: sh -c 'sleep 45 && python videoragqna.py' + entrypoint: sh -c 'sleep 45 && python videoqna.py' environment: http_proxy: ${http_proxy} https_proxy: ${https_proxy} @@ -128,11 +128,11 @@ services: LVM_SERVICE_HOST_IP: ${LVM_SERVICE_HOST_IP} ipc: host restart: always - videoragqna-xeon-ui-server: - image: ${REGISTRY:-opea}/videoragqna-ui:${TAG:-latest} - container_name: videoragqna-xeon-ui-server + videoqna-xeon-ui-server: + image: ${REGISTRY:-opea}/videoqna-ui:${TAG:-latest} + container_name: videoqna-xeon-ui-server depends_on: - - videoragqna-xeon-backend-server + - videoqna-xeon-backend-server ports: - "5173:5173" environment: diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/data/op_1_0320241830.mp4 b/VideoQnA/docker_compose/intel/cpu/xeon/data/op_1_0320241830.mp4 similarity index 100% rename from VideoRAGQnA/docker_compose/intel/cpu/xeon/data/op_1_0320241830.mp4 rename to VideoQnA/docker_compose/intel/cpu/xeon/data/op_1_0320241830.mp4 diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/set_env.sh b/VideoQnA/docker_compose/intel/cpu/xeon/set_env.sh similarity index 95% rename from VideoRAGQnA/docker_compose/intel/cpu/xeon/set_env.sh rename to VideoQnA/docker_compose/intel/cpu/xeon/set_env.sh index 8897e5709..56e894e1c 100644 --- a/VideoRAGQnA/docker_compose/intel/cpu/xeon/set_env.sh +++ b/VideoQnA/docker_compose/intel/cpu/xeon/set_env.sh @@ -12,7 +12,7 @@ export RERANK_SERVICE_HOST_IP=${host_ip} export LVM_SERVICE_HOST_IP=${host_ip} export LVM_ENDPOINT="http://${host_ip}:9009" -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/videoragqna" +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/videoqna" export BACKEND_HEALTH_CHECK_ENDPOINT="http://${host_ip}:8888/v1/health_check" export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep" export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_file" @@ -20,6 +20,6 @@ export DATAPREP_GET_VIDEO_LIST_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_ export VDMS_HOST=${host_ip} export VDMS_PORT=8001 -export INDEX_NAME="mega-videoragqna" +export INDEX_NAME="mega-videoqna" export USECLIP=1 export LLM_DOWNLOAD="True" # Set to "False" before redeploy LVM server to avoid model download diff --git a/VideoRAGQnA/docker_image_build/build.yaml b/VideoQnA/docker_image_build/build.yaml similarity index 73% rename from VideoRAGQnA/docker_image_build/build.yaml rename to VideoQnA/docker_image_build/build.yaml index 1354d378f..b8c15138d 100644 --- a/VideoRAGQnA/docker_image_build/build.yaml +++ b/VideoQnA/docker_image_build/build.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 services: - videoragqna: + videoqna: build: args: http_proxy: ${http_proxy} @@ -10,46 +10,46 @@ services: no_proxy: ${no_proxy} context: ../ dockerfile: ./Dockerfile - image: ${REGISTRY:-opea}/videoragqna:${TAG:-latest} - videoragqna-ui: + image: ${REGISTRY:-opea}/videoqna:${TAG:-latest} + videoqna-ui: build: context: ../ui dockerfile: ./docker/Dockerfile - extends: videoragqna - image: ${REGISTRY:-opea}/videoragqna-ui:${TAG:-latest} + extends: videoqna + image: ${REGISTRY:-opea}/videoqna-ui:${TAG:-latest} dataprep-multimodal-vdms: build: context: GenAIComps dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile - extends: videoragqna + extends: videoqna image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest} embedding-multimodal-clip: build: context: GenAIComps dockerfile: comps/embeddings/multimodal_clip/Dockerfile - extends: videoragqna + extends: videoqna image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest} retriever-vdms: build: context: GenAIComps dockerfile: comps/retrievers/vdms/langchain/Dockerfile - extends: videoragqna + extends: videoqna image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest} - reranking-videoragqna: + reranking-videoqna: build: context: GenAIComps - dockerfile: comps/reranks/video-rag-qna/Dockerfile - extends: videoragqna - image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest} + dockerfile: comps/reranks/videoqna/Dockerfile + extends: videoqna + image: ${REGISTRY:-opea}/reranking-videoqna:${TAG:-latest} video-llama-lvm-server: build: context: GenAIComps dockerfile: comps/lvms/video-llama/dependency/Dockerfile - extends: videoragqna + extends: videoqna image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest} lvm-video-llama: build: context: GenAIComps dockerfile: comps/lvms/video-llama/Dockerfile - extends: videoragqna + extends: videoqna image: ${REGISTRY:-opea}/lvm-video-llama:${TAG:-latest} diff --git a/VideoRAGQnA/tests/test_compose_on_xeon.sh b/VideoQnA/tests/test_compose_on_xeon.sh similarity index 93% rename from VideoRAGQnA/tests/test_compose_on_xeon.sh rename to VideoQnA/tests/test_compose_on_xeon.sh index fbaba5910..07e856bb5 100755 --- a/VideoRAGQnA/tests/test_compose_on_xeon.sh +++ b/VideoQnA/tests/test_compose_on_xeon.sh @@ -51,7 +51,7 @@ function start_services() { sleep 1m # List of containers running uvicorn - list=("dataprep-vdms-server" "embedding-multimodal-server" "retriever-vdms-server" "reranking-videoragqna-server" "video-llama-lvm-server" "lvm-video-llama" "videoragqna-xeon-backend-server") + list=("dataprep-vdms-server" "embedding-multimodal-server" "retriever-vdms-server" "reranking-videoqna-server" "video-llama-lvm-server" "lvm-video-llama" "videoqna-xeon-backend-server") # Define the maximum time limit in seconds TIME_LIMIT=5400 @@ -101,7 +101,7 @@ function start_services() { sleep 5m done - if docker logs videoragqna-xeon-ui-server 2>&1 | grep -q "Streamlit app"; then + if docker logs videoqna-xeon-ui-server 2>&1 | grep -q "Streamlit app"; then return 0 else return 1 @@ -176,7 +176,7 @@ function validate_microservices() { "${ip_address}:8000/v1/reranking" \ "video_url" \ "reranking" \ - "reranking-videoragqna-server" \ + "reranking-videoqna-server" \ '{ "retrieved_docs": [{"doc": [{"text": "retrieved text"}]}], "initial_query": "query", @@ -199,10 +199,10 @@ function validate_microservices() { function validate_megaservice() { validate_services \ - "${ip_address}:8888/v1/videoragqna" \ + "${ip_address}:8888/v1/videoqna" \ "man" \ - "videoragqna-xeon-backend-server" \ - "videoragqna-xeon-backend-server" \ + "videoqna-xeon-backend-server" \ + "videoqna-xeon-backend-server" \ '{"messages":"What is the man doing?","stream":"True"}' } @@ -216,12 +216,12 @@ function validate_frontend() { echo "Frontend Content is as expected." else echo "Frontend Content does not match the expected result: $CONTENT" - docker logs videoragqna-xeon-ui-server >> ${LOG_PATH}/ui.log + docker logs videoqna-xeon-ui-server >> ${LOG_PATH}/ui.log exit 1 fi else echo "Frontend is not running correctly. Received status was $HTTP_STATUS" - docker logs videoragqna-xeon-ui-server >> ${LOG_PATH}/ui.log + docker logs videoqna-xeon-ui-server >> ${LOG_PATH}/ui.log exit 1 fi } diff --git a/VideoRAGQnA/ui/docker/Dockerfile b/VideoQnA/ui/docker/Dockerfile similarity index 100% rename from VideoRAGQnA/ui/docker/Dockerfile rename to VideoQnA/ui/docker/Dockerfile diff --git a/VideoRAGQnA/ui/ui.py b/VideoQnA/ui/ui.py similarity index 99% rename from VideoRAGQnA/ui/ui.py rename to VideoQnA/ui/ui.py index 9f9c8724e..cfcbaf089 100644 --- a/VideoRAGQnA/ui/ui.py +++ b/VideoQnA/ui/ui.py @@ -9,7 +9,7 @@ import requests import streamlit as st -BACKEND_SERVICE_ENDPOINT = os.getenv("BACKEND_SERVICE_ENDPOINT", "http://localhost:8888/v1/videoragqna") +BACKEND_SERVICE_ENDPOINT = os.getenv("BACKEND_SERVICE_ENDPOINT", "http://localhost:8888/v1/videoqna") BACKEND_HEALTH_CHECK_ENDPOINT = os.getenv("BACKEND_HEALTH_CHECK_ENDPOINT", "http://localhost:8888/v1/health_check") @@ -137,7 +137,7 @@ def display_messages(): def main(): st.set_page_config(initial_sidebar_state="collapsed", layout="wide") - st.title("Video RAG QnA") + st.title("VideoQnA") title_alignment = """