From 26d6ea4724aeaef9fc258d79226ed15e3c325d76 Mon Sep 17 00:00:00 2001
From: Ying Chun Guo <yingchun.guo@intel.com>
Date: Thu, 6 Jun 2024 09:31:34 +0800
Subject: [PATCH] Enable new CI runner and improve manifest e2e test scripts
 (#263)

Signed-off-by: Yingchun Guo <yingchun.guo@intel.com>
---
 .github/workflows/E2E_test_with_compose.yml |  4 +--
 .github/workflows/manifest-e2e.yaml         |  4 +--
 ChatQnA/tests/test_manifest_on_xeon.sh      | 36 ++++++++++-----------
 CodeGen/tests/test_manifest_on_xeon.sh      | 19 +++++++----
 4 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/E2E_test_with_compose.yml b/.github/workflows/E2E_test_with_compose.yml
index 5e003e46b..e62a4671d 100644
--- a/.github/workflows/E2E_test_with_compose.yml
+++ b/.github/workflows/E2E_test_with_compose.yml
@@ -39,8 +39,8 @@ jobs:
         run: |
           set -xe
           merged_commit=$(git log -1 --format='%H')
-          changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${merged_commit} | \
-          grep -vE '.github|README.md|*.txt|deprecate|kubernetes|manifest')
+          changed_files="$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${merged_commit} | \
+          grep -vE '.github|README.md|*.txt|deprecate|kubernetes|manifest')" || true
           examples=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f1 | sort -u)
           run_matrix="{\"include\":["
           for example in ${examples}; do
diff --git a/.github/workflows/manifest-e2e.yaml b/.github/workflows/manifest-e2e.yaml
index 191936773..7e152c0f8 100644
--- a/.github/workflows/manifest-e2e.yaml
+++ b/.github/workflows/manifest-e2e.yaml
@@ -36,7 +36,7 @@ jobs:
         run: |
           set -xe
           changed_files="$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | \
-          grep "/kubernetes/manifests/" | \
+          grep -E "/kubernetes/manifests/|test_manifest" | \
           grep -vE '.github|deprecated|docker')" || true
           examples=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f1 | sort -u)
           run_matrix="{\"include\":["
@@ -50,7 +50,7 @@ jobs:
                       continue # skip gaudi for K8s test temporarily
                   else
                       #lower_example=$(echo "${example}" | tr '[:upper:]' '[:lower:]')
-                      run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"inspur-icx-1\"},"
+                      run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"k8s\"},"
                   fi
               done
           done
diff --git a/ChatQnA/tests/test_manifest_on_xeon.sh b/ChatQnA/tests/test_manifest_on_xeon.sh
index b9837ec46..489dedeb1 100755
--- a/ChatQnA/tests/test_manifest_on_xeon.sh
+++ b/ChatQnA/tests/test_manifest_on_xeon.sh
@@ -16,12 +16,12 @@ function init_chatqna() {
     # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
     find . -name '*.yaml' -type f -exec sed -i "s#image: opea/*#image: $IMAGE_REPO/opea/#g" {} \;
     # set huggingface token
-    find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
+    find . -name '*.yaml' -type f -exec sed -i "s#\${HUGGINGFACEHUB_API_TOKEN}#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }
 
 function install_chatqna {
     # replace namespace "default" with real namespace
-    find . -name '*.yaml' -type f -exec sed -i "s#svc.default#svc.$NAMESPACE#g" {} \;
+    find . -name '*.yaml' -type f -exec sed -i "s#default.svc#$NAMESPACE.svc#g" {} \;
     # for very yaml file in yaml_files, apply it to the k8s cluster
     yaml_files=("qna_configmap_xeon" "redis-vector-db"  "tei_embedding_service" "tei_reranking_service" "tgi_service" "retriever" "embedding" "reranking" "llm")
     for yaml_file in ${yaml_files[@]}; do
@@ -43,28 +43,28 @@ function validate_chatqna() {
     -H 'Content-Type: application/json'; do sleep 10; done
 
     # check megaservice works
+    # generate a random logfile name to avoid conflict among multiple runners
+    LOGFILE=$LOG_PATH/curlmega_$NAMESPACE.log
     curl http://chaqna-xeon-backend-server-svc.$NAMESPACE:8888/v1/chatqna -H "Content-Type: application/json" -d '{
-        "messages": "What is the revenue of Nike in 2023?"}' > ${LOG_PATH}/curl_megaservice.log
+        "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
     exit_code=$?
     if [ $exit_code -ne 0 ]; then
-        echo "Megaservice failed, please check the logs in ${LOG_PATH}!"
+        echo "Megaservice failed, please check the logs in $LOGFILE!"
         exit 1
     fi
-    echo "Response check succeed!"
 
-    # Temporarily disable response check
-    # echo "Checking response results, make sure the output is reasonable. "
-    # local status=false
-    # if [[ -f $LOG_PATH/curl_megaservice.log ]] &&
-    #     [[ $(grep -c "algorithms" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
-    #     status=true
-    # fi
-    # if [ $status == false ]; then
-    #     echo "Response check failed, please check the logs in artifacts!"
-    #     exit 1
-    # else
-    #     echo "Response check succeed!"
-    # fi
+    echo "Checking response results, make sure the output is reasonable. "
+    local status=false
+    if [[ -f $LOGFILE ]] &&
+        [[ $(grep -c "billion" $LOGFILE) != 0 ]]; then
+        status=true
+    fi
+    if [ $status == false ]; then
+        echo "Response check failed, please check the logs in artifacts!"
+        exit 1
+    else
+        echo "Response check succeed!"
+    fi
 }
 
 if [ $# -eq 0 ]; then
diff --git a/CodeGen/tests/test_manifest_on_xeon.sh b/CodeGen/tests/test_manifest_on_xeon.sh
index f0406324f..ffc71d5b7 100755
--- a/CodeGen/tests/test_manifest_on_xeon.sh
+++ b/CodeGen/tests/test_manifest_on_xeon.sh
@@ -28,15 +28,22 @@ function validate_codegen() {
     ip_address=$(kubectl get svc $SERVICE_NAME -n $NAMESPACE -o jsonpath='{.spec.clusterIP}')
     port=$(kubectl get svc $SERVICE_NAME -n $NAMESPACE -o jsonpath='{.spec.ports[0].port}')
     echo "try to curl http://${ip_address}:${port}/v1/codegen..."
+
+    # generate a random logfile name to avoid conflict among multiple runners
+    LOGFILE=$LOG_PATH/curlmega_$NAMESPACE.log
     # Curl the Mega Service
-    curl http://${ip_address}:${port}/v1/codegen -H "Content-Type: application/json" -d '{
-        "model": "ise-uiuc/Magicoder-S-DS-6.7B",
-        "messages": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' > curl_megaservice.log
+    curl http://${ip_address}:${port}/v1/codegen -H "Content-Type: application/json" \
+    -d '{"messages": "def print_hello_world():"}' > $LOGFILE
+    exit_code=$?
+    if [ $exit_code -ne 0 ]; then
+        echo "Megaservice codegen failed, please check the logs in $LOGFILE!"
+        exit 1
+    fi
 
     echo "Checking response results, make sure the output is reasonable. "
-    local status=true
-    if [[ -f curl_megaservice.log ]] && \
-    [[ $(grep -c "billion" curl_megaservice.log) != 0 ]]; then
+    local status=false
+    if [[ -f $LOGFILE ]] && \
+    [[ $(grep -c "print" $LOGFILE) != 0 ]]; then
         status=true
     fi