Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable guardrail case in helm e2e tests #384

Merged
merged 1 commit into from
Sep 4, 2024
+91 −162
Merged

Enable guardrail case in helm e2e tests #384

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions .github/workflows/_helm-e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ on:
default: "xeon"
required: true
type: string
valuefile:
default: "values"
required: true
type: string

env:
CHARTS_DIR: "helm-charts"
Expand Down Expand Up @@ -96,10 +100,7 @@ jobs:
run: |
set -xe
echo "should_cleanup=true" >> $GITHUB_ENV
value_file="values.yaml"
if [ "${{ inputs.hardware }}" == "gaudi" ]; then
value_file="gaudi-values.yaml"
fi
value_file="${{ inputs.valuefile }}.yaml"
if [[ ! -f ${{ env.CHART_FOLDER }}/${value_file} ]]; then
echo "No value file found, exist test!"
echo "skip_validate=true" >> $GITHUB_ENV
Expand Down
79 changes: 59 additions & 20 deletions .github/workflows/manual-helm-cd-workflow.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ on:
description: "Image tag to be tested"
required: true
type: string
nodes:
default: "xeon,gaudi"
required: true
opea_branch:
default: "main"
description: 'OPEA branch for image build'
required: false
type: string
description: 'Hardwares used to run tests'

env:
CHARTS_DIR: "helm-charts"
Expand All @@ -28,41 +28,80 @@ jobs:
get-build-matrix:
runs-on: ubuntu-latest
outputs:
services: ${{ steps.get-services.outputs.services }}
nodes: ${{ steps.get-services.outputs.nodes }}
run_matrix: ${{ steps.get-services.outputs.run_matrix }}
steps:
- name: Checkout out Repo
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.opea_branch }}

- name: Get test Services
id: get-services
run: |
set -x
run_matrix="{\"include\":["
if [[ -z "${{ inputs.workloads }}" ]]; then
subfolders=$(find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | sed 's/^/"/; s/$/"/' | paste -sd, -)
common_subfolders=$(find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed 's|^|"common/|; s/$/"/' | paste -sd, -)
services="[$subfolders,$common_subfolders]"
echo "services=$services" >> $GITHUB_OUTPUT
find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | while read -r subfolder; do
for file in "$CHARTS_DIR/$subfolder"/*values.yaml; do
if [ -f "$file" ]; then
if [[ "$file" == *"nv-values.yaml" ]]; then
continue
fi
filename=$(basename "$file" .yaml)
if [[ "$filename" == *"gaudi"* ]]; then
run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
else
run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
fi
fi
done
done
find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | while read -r subfolder; do
for file in "$CHARTS_DIR/common/$subfolder"/*values.yaml; do
if [ -f "$file" ]; then
if [[ "$file" == *"nv-values.yaml" ]]; then
continue
fi
filename=$(basename "$file" .yaml)
if [[ "$filename" == *"gaudi"* ]]; then
run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
else
run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
fi
fi
done
done
else
service_list=($(echo ${{ github.event.inputs.workloads }} | tr ',' ' '))
services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
echo "services=$services" >> $GITHUB_OUTPUT
for service in $service_list; do
for file in "$CHARTS_DIR/$service"/*values.yaml; do
if [ -f "$file" ]; then
if [[ "$file" == *"nv-values.yaml" ]]; then
continue
fi
filename=$(basename "$file" .yaml)
if [[ "$filename" == *"gaudi"* ]]; then
run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
else
run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
fi
fi
done
done
fi
node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' '))
nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
echo "nodes=$nodes" >> $GITHUB_OUTPUT
run_matrix=$run_matrix"]}"
echo "run_matrix=${run_matrix}"
echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
helm-release:
needs: get-build-matrix
strategy:
matrix:
workload: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }}
matrix: ${{ fromJSON(needs.get-build-matrix.outputs.run_matrix) }}
uses: ./.github/workflows/_helm-e2e.yaml
with:
tag: ${{ inputs.tag }}
workload: ${{ matrix.workload }}
hardware: ${{ matrix.node }}
workload: ${{ matrix.example }}
hardware: ${{ matrix.hardware }}
valuefile: ${{ matrix.valuefile }}
secrets: inherit
165 changes: 27 additions & 138 deletions .github/workflows/pr-chart-e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,34 @@ jobs:
cut -d'/' -f3 | sort -u )
run_matrix="{\"include\":["
for chart in ${e2e_charts}; do
if [ -f $CHARTS_DIR/$chart/gaudi-values.yaml ]; then
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\"},"
fi
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\"},"
for file in "$CHARTS_DIR/$chart"/*values.yaml; do
if [ -f "$file" ]; then
if [[ "$file" == *"nv-values.yaml" ]]; then
continue
fi
filename=$(basename "$file" .yaml)
if [[ "$filename" == *"gaudi"* ]]; then
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
else
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
fi
fi
done
done
for chart in ${common_charts}; do
if [ -f $CHARTS_DIR/common/$chart/gaudi-values.yaml ]; then
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\",\"directory\":\"common\"},"
fi
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\",\"directory\":\"common\"},"
for file in "$CHARTS_DIR/common/$chart"/*values.yaml; do
if [ -f "$file" ]; then
if [[ "$file" == *"nv-values.yaml" ]]; then
continue
fi
filename=$(basename "$file" .yaml)
if [[ "$filename" == *"gaudi"* ]]; then
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\",\"directory\":\"common\"},"
else
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\",\"directory\":\"common\"},"
fi
fi
done
done
run_matrix=$run_matrix"]}"
echo "run_matrix=${run_matrix}"
Expand All @@ -71,134 +89,5 @@ jobs:
with:
workload: ${{ matrix.directory }}/${{ matrix.example }}
hardware: ${{ matrix.hardware }}
# tag: ${{ needs.image-build.outputs.image_tag }}
# opea-branch: "main"
valuefile: ${{ matrix.valuefile }}
secrets: inherit

# needs: job1
# if: always() && ${{ needs.job1.outputs.run_matrix.example.length }} > 0
# strategy:
# matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
# runs-on: ${{ matrix.hardware }}
# continue-on-error: true
# outputs:
# should_cleanup: ${{ steps.set_boolean.outputs.should_cleanup }}
# steps:
# - name: E2e test chart
# run: |
# echo "Matrix - chart: ${{ matrix.example }}"

# - name: Clean Up Working Directory
# run: sudo rm -rf ${{github.workspace}}/*

# - name: Checkout out Repo
# uses: actions/checkout@v4
# with:
# ref: "refs/pull/${{ github.event.number }}/merge"

# - name: Set variables
# run: |
# echo "RELEASE_NAME=${{ matrix.example }}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
# echo "NAMESPACE=${{ matrix.example }}-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
# echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
# echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
# echo "should_cleanup=false" >> $GITHUB_ENV
# echo "skip_validate=false" >> $GITHUB_ENV
# echo "RELEASENAME=$RELEASE_NAME"
# echo "NAMESPACE=$NAMESPACE"
# if [ -n "${{ matrix.directory }}" ]; then
# echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.directory }}/${{ matrix.example }}" >> $GITHUB_ENV
# else
# echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.example }}" >> $GITHUB_ENV
# fi

# - name: Initialize chart testing
# run: |
# # Replace values for CI test environment
# USER_ID=$(whoami)
# CHART_MOUNT=/home/$USER_ID/.cache/huggingface/hub
# HFTOKEN=$(cat /home/$USER_ID/.cache/huggingface/token)
# pushd helm-charts
# # insert a prefix before opea/.*, the prefix is OPEA_IMAGE_REPO
# find . -name '*values.yaml' -type f -exec sed -i "s#repository: opea/*#repository: ${OPEA_IMAGE_REPO}opea/#g" {} \;
# # set OPEA image tag to latest
# find . -name '*values.yaml' -type f -exec sed -i 's#tag: ""#tag: latest#g' {} \;
# # set huggingface token
# find . -name '*values.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#${HFTOKEN}#g" {} \;
# # replace the mount dir "Volume: *" with "Volume: $CHART_MOUNT"
# find . -name '*values.yaml' -type f -exec sed -i "s#modelUseHostPath: .*#modelUseHostPath: $CHART_MOUNT#g" {} \;
# # replace the pull policy "IfNotPresent" with "Always"
# find . -name '*values.yaml' -type f -exec sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" {} \;
# popd

# - name: Helm install
# id: install
# env:
# GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
# GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
# run: |
# set -xe
# echo "should_cleanup=true" >> $GITHUB_ENV
# helm-charts/update_dependency.sh && helm dependency update ${{ env.CHART_FOLDER}}
# value_file="values.yaml"
# if [ "${{ matrix.hardware }}" == "gaudi" ]; then
# value_file="gaudi-values.yaml"
# fi
# if ! helm install --create-namespace --namespace $NAMESPACE --wait \
# --timeout "$ROLLOUT_TIMEOUT_SECONDS" \
# --set autodependency.enabled=true \
# --set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
# --set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
# --values ${{ env.CHART_FOLDER}}/${value_file} \
# $RELEASE_NAME ${{ env.CHART_FOLDER}} ; then
# echo "Failed to install chart ${{ matrix.example }}"
# echo "skip_validate=true" >> $GITHUB_ENV
# .github/workflows/scripts/e2e/chart_test.sh dump_pods_status $NAMESPACE
# exit 1
# fi

# - name: Validate e2e test
# if: always()
# run: |
# set -xe
# if $skip_validate; then
# echo "Skip validate"
# else
# LOG_PATH=/home/$(whoami)/logs
# chart=${{ matrix.example }}
# helm test -n $NAMESPACE $RELEASE_NAME --logs |tee ${LOG_PATH}/charts-${chart}.log
# exit_code=$?
# if [ $exit_code -ne 0 ]; then
# echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
# exit 1
# fi

# echo "Checking response results, make sure the output is reasonable. "
# teststatus=false
# if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
# [[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
# teststatus=false
# .github/workflows/scripts/e2e/chart_test.sh dump_all_pod_logs $NAMESPACE
# else
# teststatus=true
# fi

# if [ $teststatus == false ]; then
# echo "Response check failed, please check the logs in artifacts!"
# exit 1
# else
# echo "Response check succeed!"
# exit 0
# fi
# fi

# - name: Helm uninstall
# if: always()
# run: |
# if $should_cleanup; then
# helm uninstall $RELEASE_NAME --namespace $NAMESPACE
# if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
# kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
# kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
# fi
# fi