open chatqna guardrails test

ChatQnA/docker/chatqna_guardrails.py

@@ -66,20 +66,21 @@ def add_remote_service(self):
             use_remote_service=True,
             service_type=ServiceType.LLM,
         )
-        guardrail_out = MicroService(
-            name="guardrail_out",
-            host=GUARDRAIL_SERVICE_HOST_IP,
-            port=GUARDRAIL_SERVICE_PORT,
-            endpoint="/v1/guardrails",
-            use_remote_service=True,
-            service_type=ServiceType.GUARDRAIL,
-        )
-        self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm).add(guardrail_out)
+        # guardrail_out = MicroService(
+        #     name="guardrail_out",
+        #     host=GUARDRAIL_SERVICE_HOST_IP,
+        #     port=GUARDRAIL_SERVICE_PORT,
+        #     endpoint="/v1/guardrails",
+        #     use_remote_service=True,
+        #     service_type=ServiceType.GUARDRAIL,
+        # )
+        # self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm).add(guardrail_out)
+        self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm)
         self.megaservice.flow_to(guardrail_in, embedding)
         self.megaservice.flow_to(embedding, retriever)
         self.megaservice.flow_to(retriever, rerank)
         self.megaservice.flow_to(rerank, llm)
-        self.megaservice.flow_to(llm, guardrail_out)
+        # self.megaservice.flow_to(llm, guardrail_out)
         self.gateway = ChatQnAGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
 
 

ChatQnA/docker/docker_build_compose.yaml

@@ -10,6 +10,11 @@ services:
         no_proxy: ${no_proxy}
       dockerfile: ./Dockerfile
     image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
+  chatqna-guardrails:
+    build:
+      dockerfile: ./Dockerfile_guardrails
+    extends: chatqna
+    image: ${REGISTRY:-opea}/chatqna-guardrails:${TAG:-latest}
   chatqna-ui:
     build:
       context: ui
@@ -94,6 +99,12 @@ services:
       dockerfile: comps/dataprep/qdrant/docker/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
+  guardrails-tgi:
+    build:
+      context: GenAIComps
+      dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
+    extends: chatqna
+    image: ${REGISTRY:-opea}/guardrails-tgi:${TAG:-latest}
   tei-gaudi:
     build:
       context: tei-gaudi

ChatQnA/tests/_test_chatqna_guardrails_on_gaudi.sh → ChatQnA/tests/test_chatqna_guardrails_on_gaudi.sh

@@ -3,48 +3,33 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -e
-echo "IMAGE_REPO=${IMAGE_REPO}"
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
-    cd $WORKPATH
+    cd $WORKPATH/docker
     git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps
-
-    docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile .
-    docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile .
-    docker build -t opea/reranking-tei:latest -f comps/reranks/tei/docker/Dockerfile .
-    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
-    docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile .
-    docker build -t opea/guardrails-tgi:latest -f comps/guardrails/llama_guard/docker/Dockerfile .
+    git clone https://github.com/huggingface/tei-gaudi
 
-#    cd ..
-#    git clone https://github.com/huggingface/tei-gaudi
-#    cd tei-gaudi/
-#    docker build --no-cache -f Dockerfile-hpu -t opea/tei-gaudi:latest .
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="chatqna-guardrails chatqna-ui dataprep-redis embedding-tei retriever-redis reranking-tei llm-tgi tei-gaudi guardrails-tgi"
+    docker compose -f docker_build_compose.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
     docker pull ghcr.io/huggingface/tgi-gaudi:2.0.1
     docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
 
-    cd $WORKPATH/docker
-    docker build --no-cache -t opea/chatqna-guardrails:latest -f Dockerfile_guardrails .
-
-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile .
-
     docker images
 }
 
 function start_services() {
-    # build tei-gaudi for each test instead of pull from local registry
-    cd $WORKPATH
-    git clone https://github.com/huggingface/tei-gaudi
-    cd tei-gaudi/
-    docker build --no-cache -q -f Dockerfile-hpu -t opea/tei-gaudi:latest .
-
     cd $WORKPATH/docker/gaudi
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export RERANK_MODEL_ID="BAAI/bge-reranker-base"
@@ -69,18 +54,6 @@ function start_services() {
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env
 
-    if [[ "$IMAGE_REPO" != "" ]]; then
-        # Replace the container name with a test-specific name
-        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-        sed -i "s#image: opea/chatqna-guardrails:latest#image: opea/chatqna:${IMAGE_TAG}#g" compose_guardrails.yaml
-        sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" compose_guardrails.yaml
-        sed -i "s#image: opea/chatqna-conversation-ui:latest#image: opea/chatqna-conversation-ui:${IMAGE_TAG}#g" compose_guardrails.yaml
-        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" compose_guardrails.yaml
-        sed -i "s#image: ${IMAGE_REPO}opea/tei-gaudi:latest#image: opea/tei-gaudi:latest#g" compose_guardrails.yaml
-        echo "cat compose_guardrails.yaml"
-        cat compose_guardrails.yaml
-    fi
-
     # Start Docker Containers
     docker compose -f compose_guardrails.yaml up -d
     n=0
@@ -92,6 +65,17 @@ function start_services() {
         sleep 1s
         n=$((n+1))
     done
+
+    # Make sure tgi guardrails service is ready
+    n=0
+    until [[ "$n" -ge 400 ]]; do
+        docker logs tgi-guardrails-server > tgi_guardrails_service_start.log
+        if grep -q Connected tgi_guardrails_service_start.log; then
+            break
+        fi
+        sleep 1s
+        n=$((n+1))
+    done
 }
 
 function validate_services() {
@@ -101,24 +85,27 @@ function validate_services() {
     local DOCKER_NAME="$4"
     local INPUT_DATA="$5"
 
-    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
-    if [ "$HTTP_STATUS" -eq 200 ]; then
-        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+    HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
+    HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
+    RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
 
-        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+    docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
 
-        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-            echo "[ $SERVICE_NAME ] Content is as expected."
-        else
-            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
-            exit 1
-        fi
-    else
+    # check response status
+    if [ "$HTTP_STATUS" -ne "200" ]; then
         echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
-        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
         exit 1
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
     fi
+    # check response body
+    if [[ "$RESPONSE_BODY" != *"$EXPECTED_RESULT"* ]]; then
+        echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
+        exit 1
+    else
+        echo "[ $SERVICE_NAME ] Content is as expected."
+    fi
+
     sleep 1s
 }
 
@@ -128,15 +115,15 @@ function validate_microservices() {
     # tei for embedding service
     validate_services \
         "${ip_address}:8090/embed" \
-        "\[\[" \
+        "[[" \
         "tei-embedding" \
         "tei-embedding-gaudi-server" \
         '{"inputs":"What is Deep Learning?"}'
 
     # embedding microservice
     validate_services \
         "${ip_address}:6000/v1/embeddings" \
-        '"text":"What is Deep Learning?","embedding":\[' \
+        '"text":"What is Deep Learning?","embedding":[' \
         "embedding" \
         "embedding-tei-server" \
         '{"text":"What is Deep Learning?"}'
@@ -147,7 +134,7 @@ function validate_microservices() {
     test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
     validate_services \
         "${ip_address}:7000/v1/retrieval" \
-        " " \
+        "retrieved_docs" \
         "retrieval" \
         "retriever-redis-server" \
         "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}"
@@ -186,7 +173,7 @@ function validate_microservices() {
 
     # tgi for guardrails service
     validate_services \
-        "${ip_address}:8008/generate" \
+        "${ip_address}:8088/generate" \
         "generated_text" \
         "tgi-guardrails" \
         "tgi-guardrails-server" \
@@ -217,13 +204,16 @@ function validate_frontend() {
     cd $WORKPATH/docker/ui/svelte
     local conda_env_name="OPEA_e2e"
     export PATH=${HOME}/miniforge3/bin/:$PATH
-#    conda remove -n ${conda_env_name} --all -y
-#    conda create -n ${conda_env_name} python=3.12 -y
+    if conda info --envs | grep -q "$conda_env_name"; then
+        echo "$conda_env_name exist!"
+    else
+        conda create -n ${conda_env_name} python=3.12 -y
+    fi
     source activate ${conda_env_name}
 
     sed -i "s/localhost/$ip_address/g" playwright.config.ts
 
-#    conda install -c conda-forge nodejs -y
+    conda install -c conda-forge nodejs -y
     npm install && npm ci && npx playwright install --with-deps
     node -v && npm -v && pip list
 
@@ -246,7 +236,7 @@ function stop_docker() {
 function main() {
 
     stop_docker
-    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
     start_time=$(date +%s)
     start_services
     end_time=$(date +%s)