From 5b638ebc2e3fa32608f060e13eb9a6ce5e399c36 Mon Sep 17 00:00:00 2001 From: Yingchun Guo Date: Tue, 3 Sep 2024 14:15:52 +0800 Subject: [PATCH] enable guardrail case in helm e2e tests Signed-off-by: Yingchun Guo --- .github/workflows/_helm-e2e.yaml | 9 +- .../workflows/manual-helm-cd-workflow.yaml | 79 ++++++--- .github/workflows/pr-chart-e2e.yaml | 165 +++--------------- 3 files changed, 91 insertions(+), 162 deletions(-) diff --git a/.github/workflows/_helm-e2e.yaml b/.github/workflows/_helm-e2e.yaml index b54e163f..8f470e31 100644 --- a/.github/workflows/_helm-e2e.yaml +++ b/.github/workflows/_helm-e2e.yaml @@ -19,6 +19,10 @@ on: default: "xeon" required: true type: string + valuefile: + default: "values" + required: true + type: string env: CHARTS_DIR: "helm-charts" @@ -96,10 +100,7 @@ jobs: run: | set -xe echo "should_cleanup=true" >> $GITHUB_ENV - value_file="values.yaml" - if [ "${{ inputs.hardware }}" == "gaudi" ]; then - value_file="gaudi-values.yaml" - fi + value_file="${{ inputs.valuefile }}.yaml" if [[ ! -f ${{ env.CHART_FOLDER }}/${value_file} ]]; then echo "No value file found, exist test!" echo "skip_validate=true" >> $GITHUB_ENV diff --git a/.github/workflows/manual-helm-cd-workflow.yaml b/.github/workflows/manual-helm-cd-workflow.yaml index d4eed4cc..77299305 100644 --- a/.github/workflows/manual-helm-cd-workflow.yaml +++ b/.github/workflows/manual-helm-cd-workflow.yaml @@ -15,11 +15,11 @@ on: description: "Image tag to be tested" required: true type: string - nodes: - default: "xeon,gaudi" - required: true + opea_branch: + default: "main" + description: 'OPEA branch for image build' + required: false type: string - description: 'Hardwares used to run tests' env: CHARTS_DIR: "helm-charts" @@ -28,41 +28,80 @@ jobs: get-build-matrix: runs-on: ubuntu-latest outputs: - services: ${{ steps.get-services.outputs.services }} - nodes: ${{ steps.get-services.outputs.nodes }} + run_matrix: ${{ steps.get-services.outputs.run_matrix }} steps: - name: Checkout out Repo uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ inputs.opea_branch }} - name: Get test Services id: get-services run: | set -x + run_matrix="{\"include\":[" if [[ -z "${{ inputs.workloads }}" ]]; then - subfolders=$(find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | sed 's/^/"/; s/$/"/' | paste -sd, -) - common_subfolders=$(find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed 's|^|"common/|; s/$/"/' | paste -sd, -) - services="[$subfolders,$common_subfolders]" - echo "services=$services" >> $GITHUB_OUTPUT + find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | while read -r subfolder; do + for file in "$CHARTS_DIR/$subfolder"/*values.yaml; do + if [ -f "$file" ]; then + if [[ "$file" == *"nv-values.yaml" ]]; then + continue + fi + filename=$(basename "$file" .yaml) + if [[ "$filename" == *"gaudi"* ]]; then + run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"}," + else + run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"}," + fi + fi + done + done + find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | while read -r subfolder; do + for file in "$CHARTS_DIR/common/$subfolder"/*values.yaml; do + if [ -f "$file" ]; then + if [[ "$file" == *"nv-values.yaml" ]]; then + continue + fi + filename=$(basename "$file" .yaml) + if [[ "$filename" == *"gaudi"* ]]; then + run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"}," + else + run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"}," + fi + fi + done + done else service_list=($(echo ${{ github.event.inputs.workloads }} | tr ',' ' ')) - services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.') - echo "services=$services" >> $GITHUB_OUTPUT + for service in $service_list; do + for file in "$CHARTS_DIR/$service"/*values.yaml; do + if [ -f "$file" ]; then + if [[ "$file" == *"nv-values.yaml" ]]; then + continue + fi + filename=$(basename "$file" .yaml) + if [[ "$filename" == *"gaudi"* ]]; then + run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"}," + else + run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"}," + fi + fi + done + done fi - node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' ')) - nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.') - echo "nodes=$nodes" >> $GITHUB_OUTPUT + run_matrix=$run_matrix"]}" + echo "run_matrix=${run_matrix}" + echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT helm-release: needs: get-build-matrix strategy: - matrix: - workload: ${{ fromJSON(needs.get-build-matrix.outputs.services) }} - node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }} + matrix: ${{ fromJSON(needs.get-build-matrix.outputs.run_matrix) }} uses: ./.github/workflows/_helm-e2e.yaml with: tag: ${{ inputs.tag }} - workload: ${{ matrix.workload }} - hardware: ${{ matrix.node }} + workload: ${{ matrix.example }} + hardware: ${{ matrix.hardware }} + valuefile: ${{ matrix.valuefile }} secrets: inherit diff --git a/.github/workflows/pr-chart-e2e.yaml b/.github/workflows/pr-chart-e2e.yaml index d31f3372..691f7f93 100644 --- a/.github/workflows/pr-chart-e2e.yaml +++ b/.github/workflows/pr-chart-e2e.yaml @@ -47,16 +47,34 @@ jobs: cut -d'/' -f3 | sort -u ) run_matrix="{\"include\":[" for chart in ${e2e_charts}; do - if [ -f $CHARTS_DIR/$chart/gaudi-values.yaml ]; then - run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\"}," - fi - run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\"}," + for file in "$CHARTS_DIR/$chart"/*values.yaml; do + if [ -f "$file" ]; then + if [[ "$file" == *"nv-values.yaml" ]]; then + continue + fi + filename=$(basename "$file" .yaml) + if [[ "$filename" == *"gaudi"* ]]; then + run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"}," + else + run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"}," + fi + fi + done done for chart in ${common_charts}; do - if [ -f $CHARTS_DIR/common/$chart/gaudi-values.yaml ]; then - run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\",\"directory\":\"common\"}," - fi - run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\",\"directory\":\"common\"}," + for file in "$CHARTS_DIR/common/$chart"/*values.yaml; do + if [ -f "$file" ]; then + if [[ "$file" == *"nv-values.yaml" ]]; then + continue + fi + filename=$(basename "$file" .yaml) + if [[ "$filename" == *"gaudi"* ]]; then + run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\",\"directory\":\"common\"}," + else + run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\",\"directory\":\"common\"}," + fi + fi + done done run_matrix=$run_matrix"]}" echo "run_matrix=${run_matrix}" @@ -71,134 +89,5 @@ jobs: with: workload: ${{ matrix.directory }}/${{ matrix.example }} hardware: ${{ matrix.hardware }} - # tag: ${{ needs.image-build.outputs.image_tag }} - # opea-branch: "main" + valuefile: ${{ matrix.valuefile }} secrets: inherit - - # needs: job1 - # if: always() && ${{ needs.job1.outputs.run_matrix.example.length }} > 0 - # strategy: - # matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }} - # runs-on: ${{ matrix.hardware }} - # continue-on-error: true - # outputs: - # should_cleanup: ${{ steps.set_boolean.outputs.should_cleanup }} - # steps: - # - name: E2e test chart - # run: | - # echo "Matrix - chart: ${{ matrix.example }}" - - # - name: Clean Up Working Directory - # run: sudo rm -rf ${{github.workspace}}/* - - # - name: Checkout out Repo - # uses: actions/checkout@v4 - # with: - # ref: "refs/pull/${{ github.event.number }}/merge" - - # - name: Set variables - # run: | - # echo "RELEASE_NAME=${{ matrix.example }}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV - # echo "NAMESPACE=${{ matrix.example }}-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV - # echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV - # echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV - # echo "should_cleanup=false" >> $GITHUB_ENV - # echo "skip_validate=false" >> $GITHUB_ENV - # echo "RELEASENAME=$RELEASE_NAME" - # echo "NAMESPACE=$NAMESPACE" - # if [ -n "${{ matrix.directory }}" ]; then - # echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.directory }}/${{ matrix.example }}" >> $GITHUB_ENV - # else - # echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.example }}" >> $GITHUB_ENV - # fi - - # - name: Initialize chart testing - # run: | - # # Replace values for CI test environment - # USER_ID=$(whoami) - # CHART_MOUNT=/home/$USER_ID/.cache/huggingface/hub - # HFTOKEN=$(cat /home/$USER_ID/.cache/huggingface/token) - # pushd helm-charts - # # insert a prefix before opea/.*, the prefix is OPEA_IMAGE_REPO - # find . -name '*values.yaml' -type f -exec sed -i "s#repository: opea/*#repository: ${OPEA_IMAGE_REPO}opea/#g" {} \; - # # set OPEA image tag to latest - # find . -name '*values.yaml' -type f -exec sed -i 's#tag: ""#tag: latest#g' {} \; - # # set huggingface token - # find . -name '*values.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#${HFTOKEN}#g" {} \; - # # replace the mount dir "Volume: *" with "Volume: $CHART_MOUNT" - # find . -name '*values.yaml' -type f -exec sed -i "s#modelUseHostPath: .*#modelUseHostPath: $CHART_MOUNT#g" {} \; - # # replace the pull policy "IfNotPresent" with "Always" - # find . -name '*values.yaml' -type f -exec sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" {} \; - # popd - - # - name: Helm install - # id: install - # env: - # GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} - # GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - # run: | - # set -xe - # echo "should_cleanup=true" >> $GITHUB_ENV - # helm-charts/update_dependency.sh && helm dependency update ${{ env.CHART_FOLDER}} - # value_file="values.yaml" - # if [ "${{ matrix.hardware }}" == "gaudi" ]; then - # value_file="gaudi-values.yaml" - # fi - # if ! helm install --create-namespace --namespace $NAMESPACE --wait \ - # --timeout "$ROLLOUT_TIMEOUT_SECONDS" \ - # --set autodependency.enabled=true \ - # --set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \ - # --set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \ - # --values ${{ env.CHART_FOLDER}}/${value_file} \ - # $RELEASE_NAME ${{ env.CHART_FOLDER}} ; then - # echo "Failed to install chart ${{ matrix.example }}" - # echo "skip_validate=true" >> $GITHUB_ENV - # .github/workflows/scripts/e2e/chart_test.sh dump_pods_status $NAMESPACE - # exit 1 - # fi - - # - name: Validate e2e test - # if: always() - # run: | - # set -xe - # if $skip_validate; then - # echo "Skip validate" - # else - # LOG_PATH=/home/$(whoami)/logs - # chart=${{ matrix.example }} - # helm test -n $NAMESPACE $RELEASE_NAME --logs |tee ${LOG_PATH}/charts-${chart}.log - # exit_code=$? - # if [ $exit_code -ne 0 ]; then - # echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!" - # exit 1 - # fi - - # echo "Checking response results, make sure the output is reasonable. " - # teststatus=false - # if [[ -f $LOG_PATH/charts-${chart}.log ]] && \ - # [[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then - # teststatus=false - # .github/workflows/scripts/e2e/chart_test.sh dump_all_pod_logs $NAMESPACE - # else - # teststatus=true - # fi - - # if [ $teststatus == false ]; then - # echo "Response check failed, please check the logs in artifacts!" - # exit 1 - # else - # echo "Response check succeed!" - # exit 0 - # fi - # fi - - # - name: Helm uninstall - # if: always() - # run: | - # if $should_cleanup; then - # helm uninstall $RELEASE_NAME --namespace $NAMESPACE - # if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then - # kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all - # kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS - # fi - # fi