enable guardrail case in helm e2e tests (#384)

Signed-off-by: Yingchun Guo <[email protected]>
opea-project · Sep 4, 2024 · 491c2e2 · 491c2e2
1 parent 59000c5
commit 491c2e2
Show file tree

Hide file tree

Showing 3 changed files with 91 additions and 162 deletions.
diff --git a/.github/workflows/_helm-e2e.yaml b/.github/workflows/_helm-e2e.yaml
@@ -19,6 +19,10 @@ on:
         default: "xeon"
         required: true
         type: string
+      valuefile:
+        default: "values"
+        required: true
+        type: string
 
 env:
   CHARTS_DIR: "helm-charts"
@@ -96,10 +100,7 @@ jobs:
         run: |
           set -xe
           echo "should_cleanup=true" >> $GITHUB_ENV
-          value_file="values.yaml"
-          if [ "${{ inputs.hardware }}" == "gaudi" ]; then
-            value_file="gaudi-values.yaml"
-          fi
+          value_file="${{ inputs.valuefile }}.yaml"
           if [[ ! -f ${{ env.CHART_FOLDER }}/${value_file} ]]; then
             echo "No value file found, exist test!"
             echo "skip_validate=true" >> $GITHUB_ENV

diff --git a/.github/workflows/manual-helm-cd-workflow.yaml b/.github/workflows/manual-helm-cd-workflow.yaml
@@ -15,11 +15,11 @@ on:
         description: "Image tag to be tested"
         required: true
         type: string
-      nodes:
-        default: "xeon,gaudi"
-        required: true
+      opea_branch:
+        default: "main"
+        description: 'OPEA branch for image build'
+        required: false
         type: string
-        description: 'Hardwares used to run tests'
 
 env:
   CHARTS_DIR: "helm-charts"
@@ -28,41 +28,80 @@ jobs:
   get-build-matrix:
     runs-on: ubuntu-latest
     outputs:
-      services: ${{ steps.get-services.outputs.services }}
-      nodes: ${{ steps.get-services.outputs.nodes }}
+      run_matrix: ${{ steps.get-services.outputs.run_matrix }}
     steps:
       - name: Checkout out Repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ inputs.opea_branch }}
 
       - name: Get test Services
         id: get-services
         run: |
           set -x
+          run_matrix="{\"include\":["
           if [[ -z "${{ inputs.workloads }}" ]]; then
-            subfolders=$(find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | sed 's/^/"/; s/$/"/' | paste -sd, -)
-            common_subfolders=$(find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed 's|^|"common/|; s/$/"/' | paste -sd, -)
-            services="[$subfolders,$common_subfolders]"
-            echo "services=$services" >> $GITHUB_OUTPUT
+            find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | while read -r subfolder; do
+              for file in "$CHARTS_DIR/$subfolder"/*values.yaml; do
+                if [ -f "$file" ]; then
+                  if [[ "$file" == *"nv-values.yaml" ]]; then
+                    continue
+                  fi
+                  filename=$(basename "$file" .yaml)
+                  if [[ "$filename" == *"gaudi"* ]]; then
+                    run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                  else
+                    run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                  fi
+                fi
+              done
+            done
+            find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | while read -r subfolder; do
+              for file in "$CHARTS_DIR/common/$subfolder"/*values.yaml; do
+                if [ -f "$file" ]; then
+                  if [[ "$file" == *"nv-values.yaml" ]]; then
+                    continue
+                  fi
+                  filename=$(basename "$file" .yaml)
+                  if [[ "$filename" == *"gaudi"* ]]; then
+                    run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                  else
+                    run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                  fi
+                fi
+              done
+            done
           else
             service_list=($(echo ${{ github.event.inputs.workloads }} | tr ',' ' '))
-            services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
-            echo "services=$services" >> $GITHUB_OUTPUT
+            for service in $service_list; do
+              for file in "$CHARTS_DIR/$service"/*values.yaml; do
+                if [ -f "$file" ]; then
+                  if [[ "$file" == *"nv-values.yaml" ]]; then
+                    continue
+                  fi
+                  filename=$(basename "$file" .yaml)
+                  if [[ "$filename" == *"gaudi"* ]]; then
+                    run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                  else
+                    run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                  fi
+                fi
+              done
+            done
           fi
-          node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' '))
-          nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
-          echo "nodes=$nodes" >> $GITHUB_OUTPUT
+          run_matrix=$run_matrix"]}"
+          echo "run_matrix=${run_matrix}"
+          echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
 
   helm-release:
     needs: get-build-matrix
     strategy:
-      matrix:
-        workload: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
-        node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }}
+      matrix: ${{ fromJSON(needs.get-build-matrix.outputs.run_matrix) }}
     uses: ./.github/workflows/_helm-e2e.yaml
     with:
       tag: ${{ inputs.tag }}
-      workload: ${{ matrix.workload }}
-      hardware: ${{ matrix.node }}
+      workload: ${{ matrix.example }}
+      hardware: ${{ matrix.hardware }}
+      valuefile: ${{ matrix.valuefile }}
     secrets: inherit
diff --git a/.github/workflows/pr-chart-e2e.yaml b/.github/workflows/pr-chart-e2e.yaml
@@ -47,16 +47,34 @@ jobs:
           cut -d'/' -f3 | sort -u )
           run_matrix="{\"include\":["
           for chart in ${e2e_charts}; do
-            if [ -f $CHARTS_DIR/$chart/gaudi-values.yaml ]; then
-              run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\"},"
-            fi
-            run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\"},"
+            for file in "$CHARTS_DIR/$chart"/*values.yaml; do
+              if [ -f "$file" ]; then
+                if [[ "$file" == *"nv-values.yaml" ]]; then
+                  continue
+                fi
+                filename=$(basename "$file" .yaml)
+                if [[ "$filename" == *"gaudi"* ]]; then
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                else
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                fi
+              fi
+            done
           done
           for chart in ${common_charts}; do
-            if [ -f $CHARTS_DIR/common/$chart/gaudi-values.yaml ]; then
-              run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\",\"directory\":\"common\"},"
-            fi
-            run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\",\"directory\":\"common\"},"
+            for file in "$CHARTS_DIR/common/$chart"/*values.yaml; do
+              if [ -f "$file" ]; then
+                if [[ "$file" == *"nv-values.yaml" ]]; then
+                  continue
+                fi
+                filename=$(basename "$file" .yaml)
+                if [[ "$filename" == *"gaudi"* ]]; then
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\",\"directory\":\"common\"},"
+                else
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\",\"directory\":\"common\"},"
+                fi
+              fi
+            done
           done
           run_matrix=$run_matrix"]}"
           echo "run_matrix=${run_matrix}"
@@ -71,134 +89,5 @@ jobs:
     with:
       workload: ${{ matrix.directory }}/${{ matrix.example }}
       hardware: ${{ matrix.hardware }}
-      # tag: ${{ needs.image-build.outputs.image_tag }}
-      # opea-branch: "main"
+      valuefile: ${{ matrix.valuefile }}
     secrets: inherit
-
-    # needs: job1
-    # if: always() && ${{ needs.job1.outputs.run_matrix.example.length }} > 0
-    # strategy:
-    #   matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
-    # runs-on: ${{ matrix.hardware }}
-    # continue-on-error: true
-    # outputs:
-    #   should_cleanup: ${{ steps.set_boolean.outputs.should_cleanup }}
-    # steps:
-    #   - name: E2e test chart
-    #     run: |
-    #       echo "Matrix - chart: ${{ matrix.example }}"
-
-    #   - name: Clean Up Working Directory
-    #     run: sudo rm -rf ${{github.workspace}}/*
-
-    #   - name: Checkout out Repo
-    #     uses: actions/checkout@v4
-    #     with:
-    #       ref: "refs/pull/${{ github.event.number }}/merge"
-
-    #   - name: Set variables
-    #     run: |
-    #       echo "RELEASE_NAME=${{ matrix.example }}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
-    #       echo "NAMESPACE=${{ matrix.example }}-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
-    #       echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
-    #       echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
-    #       echo "should_cleanup=false" >> $GITHUB_ENV
-    #       echo "skip_validate=false" >> $GITHUB_ENV
-    #       echo "RELEASENAME=$RELEASE_NAME"
-    #       echo "NAMESPACE=$NAMESPACE"
-    #       if [ -n "${{ matrix.directory }}" ]; then
-    #         echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.directory }}/${{ matrix.example }}" >> $GITHUB_ENV
-    #       else
-    #         echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.example }}" >> $GITHUB_ENV
-    #       fi
-
-    #   - name: Initialize chart testing
-    #     run: |
-    #       # Replace values for CI test environment
-    #       USER_ID=$(whoami)
-    #       CHART_MOUNT=/home/$USER_ID/.cache/huggingface/hub
-    #       HFTOKEN=$(cat /home/$USER_ID/.cache/huggingface/token)
-    #       pushd helm-charts
-    #       # insert a prefix before opea/.*, the prefix is OPEA_IMAGE_REPO
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#repository: opea/*#repository: ${OPEA_IMAGE_REPO}opea/#g" {} \;
-    #       # set OPEA image tag to latest
-    #       find . -name '*values.yaml' -type f -exec sed -i 's#tag: ""#tag: latest#g' {} \;
-    #       # set huggingface token
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#${HFTOKEN}#g" {} \;
-    #       # replace the mount dir "Volume: *" with "Volume: $CHART_MOUNT"
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#modelUseHostPath: .*#modelUseHostPath: $CHART_MOUNT#g" {} \;
-    #       # replace the pull policy "IfNotPresent" with "Always"
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" {} \;
-    #       popd
-
-    #   - name: Helm install
-    #     id: install
-    #     env:
-    #       GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
-    #       GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
-    #     run: |
-    #       set -xe
-    #       echo "should_cleanup=true" >> $GITHUB_ENV
-    #       helm-charts/update_dependency.sh && helm dependency update ${{ env.CHART_FOLDER}}
-    #       value_file="values.yaml"
-    #       if [ "${{ matrix.hardware }}" == "gaudi" ]; then
-    #         value_file="gaudi-values.yaml"
-    #       fi
-    #       if ! helm install --create-namespace --namespace $NAMESPACE --wait \
-    #           --timeout "$ROLLOUT_TIMEOUT_SECONDS" \
-    #           --set autodependency.enabled=true \
-    #           --set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
-    #           --set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
-    #           --values ${{ env.CHART_FOLDER}}/${value_file} \
-    #           $RELEASE_NAME ${{ env.CHART_FOLDER}} ; then
-    #         echo "Failed to install chart ${{ matrix.example }}"
-    #         echo "skip_validate=true" >> $GITHUB_ENV
-    #         .github/workflows/scripts/e2e/chart_test.sh dump_pods_status $NAMESPACE
-    #         exit 1
-    #       fi
-
-    #   - name: Validate e2e test
-    #     if: always()
-    #     run: |
-    #       set -xe
-    #       if $skip_validate; then
-    #         echo "Skip validate"
-    #       else
-    #         LOG_PATH=/home/$(whoami)/logs
-    #         chart=${{ matrix.example }}
-    #         helm test -n $NAMESPACE $RELEASE_NAME --logs |tee ${LOG_PATH}/charts-${chart}.log
-    #         exit_code=$?
-    #         if [ $exit_code -ne 0 ]; then
-    #             echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
-    #             exit 1
-    #         fi
-
-    #         echo "Checking response results, make sure the output is reasonable. "
-    #         teststatus=false
-    #         if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
-    #         [[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
-    #             teststatus=false
-    #             .github/workflows/scripts/e2e/chart_test.sh dump_all_pod_logs $NAMESPACE
-    #         else
-    #             teststatus=true
-    #         fi
-
-    #         if [ $teststatus == false ]; then
-    #             echo "Response check failed, please check the logs in artifacts!"
-    #             exit 1
-    #         else
-    #             echo "Response check succeed!"
-    #             exit 0
-    #         fi
-    #       fi
-
-    #   - name: Helm uninstall
-    #     if: always()
-    #     run: |
-    #       if $should_cleanup; then
-    #         helm uninstall $RELEASE_NAME --namespace $NAMESPACE
-    #         if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
-    #           kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
-    #           kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
-    #         fi
-    #       fi