Skip to content

Commit

Permalink
K8s: Add test results for autoscaling
Browse files Browse the repository at this point in the history 
Signed-off-by: Viet Nguyen Duc <[email protected]>
  • Loading branch information
@VietND96
VietND96 committed Dec 3, 2024
1 parent cf0c839 commit b52e28b
Show file tree
Hide file tree
Showing 24 changed files with 779 additions and 221 deletions.
215 changes: 155 additions & 60 deletions .github/workflows/k8s-scaling-test.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Test Autoscaling
name: Test Grid Autoscaling in Kubernetes

on:
workflow_call:
Expand All @@ -8,20 +8,82 @@ on:
required: false
type: string
default: 'false'
push-results:
description: 'Publish the results to the repository'
required: false
type: boolean
default: false
iteration:
description: 'Test a specific iteration'
required: false
type: string
default: '20'
workflow_dispatch:
inputs:
publish-results:
description: 'Publish the results to the repository'
required: false
type: boolean
default: false
pr-results:
description: 'Create a PR with the results'
required: false
type: boolean
default: false
iteration:
description: 'Test a specific iteration'
required: false
type: string
default: '20'

permissions:
contents: read
contents: write
pull-requests: write

env:
RUN_ID: ${{ github.run_id }}
TEST_AUTOSCALING_ITERATIONS: ${{ github.event.inputs.iteration || '20' }}

jobs:
build-and-test:
name: Test K8s
runs-on: blacksmith-16vcpu-ubuntu-2204
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- k8s-version: 'v1.31.2'
test-strategy: test_k8s_autoscaling_job_count_strategy_default_in_chaos
cluster: 'minikube'
helm-version: 'v3.16.3'
docker-version: '27.3.1'
python-version: '3.13'
- k8s-version: 'v1.31.2'
test-strategy: test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions
cluster: 'minikube'
helm-version: 'v3.16.3'
docker-version: '27.3.1'
python-version: '3.13'
- k8s-version: 'v1.31.2'
test-strategy: test_k8s_autoscaling_job_count_strategy_default
cluster: 'minikube'
helm-version: 'v3.16.3'
docker-version: '27.3.1'
python-version: '3.13'
- k8s-version: 'v1.31.2'
test-strategy: test_k8s_autoscaling_deployment_count_in_chaos
cluster: 'minikube'
helm-version: 'v3.16.3'
docker-version: '27.3.1'
python-version: '3.13'
- k8s-version: 'v1.31.2'
test-strategy: test_k8s_autoscaling_deployment_count_with_node_max_sessions
cluster: 'minikube'
helm-version: 'v3.16.3'
docker-version: '27.3.1'
python-version: '3.13'
- k8s-version: 'v1.31.2'
test-strategy: test_k8s_autoscaling_deployment_count
cluster: 'minikube'
helm-version: 'v3.16.3'
docker-version: '27.3.1'
Expand Down Expand Up @@ -53,8 +115,6 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
check-latest: true
- name: Verify chart configuration up-to-date
run: make lint_readme_charts
- name: Get branch name (only for push to branch)
if: github.event_name == 'push'
run: echo "BRANCH=$(echo ${PUSH_BRANCH##*/})" >> $GITHUB_ENV
Expand All @@ -79,83 +139,118 @@ jobs:
echo "AUTHORS=${AUTHORS}" >> $GITHUB_ENV
env:
AUTHORS: ${{ vars.AUTHORS || 'SeleniumHQ' }}
- name: Build Helm charts
run: |
BUILD_DATE=${BUILD_DATE} make chart_build
echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV
echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV
- name: Build Docker images
uses: nick-invision/retry@master
with:
timeout_minutes: 12
max_attempts: 3
retry_wait_seconds: 60
command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
- name: Setup Kubernetes cluster
uses: nick-invision/retry@master
with:
timeout_minutes: 10
max_attempts: 3
command: CLUSTER=${CLUSTER} SERVICE_MESH=${SERVICE_MESH} KUBERNETES_VERSION=${KUBERNETES_VERSION} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
- name: Test Selenium Grid on Kubernetes with Autoscaling
uses: nick-invision/retry@master
with:
timeout_minutes: 30
max_attempts: 3
command: |
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_job_count_chaos
- name: Upload results
if: always()
uses: actions/upload-artifact@main
with:
name: chart_test_autoscaling_job_count_chaos
path: ./tests/tests/*.md
if-no-files-found: ignore
- name: Test Selenium Grid on Kubernetes with Autoscaling
uses: nick-invision/retry@master
with:
timeout_minutes: 30
max_attempts: 3
command: |
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_job_count_max_sessions
- name: Upload results
if: always()
uses: actions/upload-artifact@main
with:
name: chart_test_autoscaling_job_count_max_sessions
path: ./tests/tests/*.md
if-no-files-found: ignore
- name: Test Selenium Grid on Kubernetes with Autoscaling
- name: Build Docker images
uses: nick-invision/retry@master
with:
timeout_minutes: 30
timeout_minutes: 12
max_attempts: 3
command: |
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_job_count_strategy_accurate
- name: Upload results
if: always()
uses: actions/upload-artifact@main
with:
name: chart_test_autoscaling_job_count_strategy_accurate
path: ./tests/tests/*.md
if-no-files-found: ignore
retry_wait_seconds: 60
command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
- name: Build Helm charts
run: |
BUILD_DATE=${BUILD_DATE} make chart_build
echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV
echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV
- name: Test Selenium Grid on Kubernetes with Autoscaling
uses: nick-invision/retry@master
with:
timeout_minutes: 30
max_attempts: 3
command: |
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_job_count
NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false TEST_AUTOSCALING_ITERATIONS=${TEST_AUTOSCALING_ITERATIONS} \
make ${{ matrix.test-strategy }}
- name: Rename results
run: mv ./tests/tests/autoscaling_results.md ./tests/tests/results_${{ matrix.test-strategy }}.md
- name: Upload results
if: always()
uses: actions/upload-artifact@main
uses: actions/upload-artifact@v4
with:
name: chart_test_autoscaling_job_count
path: ./tests/tests/*.md
name: results_${{ matrix.test-strategy }}
path: ./tests/tests/results_${{ matrix.test-strategy }}.md
if-no-files-found: ignore
- name: Cleanup Kubernetes cluster
if: always()
run: CLUSTER=${CLUSTER} make chart_cluster_cleanup
- name: Clean up Docker
if: always()
run: docker system prune -af

publish-results:
name: Publish Results
if: (!failure() && !cancelled() && (github.event.inputs.publish-results == 'true'))
runs-on: ubuntu-latest
needs: build-and-test
steps:
- name: Checkout code
uses: actions/checkout@main
with:
persist-credentials: false
fetch-depth: 0
- name: Download results
uses: actions/download-artifact@v4
with:
path: ./.keda
pattern: 'results_*'
merge-multiple: 'true'
run-id: ${{ env.RUN_ID }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Commit files
run: |
git config --local user.email "[email protected]"
git config --local user.name "Selenium CI Bot"
git add .keda/.
git commit -m "[ci] Upload autoscaling in K8s test results [skip ci]" -a
- name: Push changes
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.SELENIUM_CI_TOKEN }}
branch: ${{ env.BRANCH_NAME }}
env:
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}


pr-results:
name: Create a PR with the results
if: (!failure() && !cancelled() && (github.event.inputs.pr-results == 'true'))
runs-on: ubuntu-latest
needs: build-and-test
steps:
- name: Checkout code
uses: actions/checkout@main
with:
persist-credentials: false
fetch-depth: 0
- name: Download results
uses: actions/download-artifact@v4
with:
path: ./.keda
pattern: 'results_*'
merge-multiple: 'true'
run-id: ${{ env.RUN_ID }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Commit configs
run: |
git config --local user.email "[email protected]"
git config --local user.name "Selenium CI Bot"
- name: Create Pull Request
id: cpr
uses: peter-evans/create-pull-request@main
with:
token: ${{ secrets.SELENIUM_CI_TOKEN }}
commit-message: "[ci] Upload autoscaling in K8s test results"
title: "[ci] Upload autoscaling in K8s test results"
body: "This PR contains the results of the autoscaling tests in Kubernetes"
committer: 'Selenium CI Bot <[email protected]>'
author: 'Selenium CI Bot <[email protected]>'
branch: autoscaling-results
- name: Check outputs
if: ${{ steps.cpr.outputs.pull-request-number }}
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
17 changes: 14 additions & 3 deletions .keda/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,24 @@ You can involve to review and discuss the pull requests to help us early detect

- https://github.com/kedacore/keda/pull/6368

~~- https://github.com/kedacore/keda/pull/6169 (merged, v2.16.0)~~
- ~~https://github.com/kedacore/keda/pull/6169 (merged, v2.16.0)~~

[kedacore/keda-docs](https://github.com/kedacore/keda-docs)

- https://github.com/kedacore/keda-docs/pull/1504
- ~~https://github.com/kedacore/keda-docs/pull/1468 (merged, v2.16.0)~~

~~- https://github.com/kedacore/keda-docs/pull/1468 (merged, v2.16.0)~~
# Test results of the patch scaler

There are tests for the patched scaler implementation. You can run the tests by following the steps in [../tests/README.md](../tests/README.md).

Test results could be referred to

- [results_test_k8s_autoscaling_job_count_strategy_default.md](./results_test_k8s_autoscaling_job_count_strategy_default.md)
- [results_test_k8s_autoscaling_job_count_strategy_default_in_chaos.md](./results_test_k8s_autoscaling_job_count_strategy_default_in_chaos.md)
- [results_test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions.md](./results_test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions.md)
- [results_test_k8s_autoscaling_deployment_count.md](./results_test_k8s_autoscaling_deployment_count.md)
- [results_test_k8s_autoscaling_deployment_count_in_chaos.md](./results_test_k8s_autoscaling_deployment_count_in_chaos.md)
- [results_test_k8s_autoscaling_deployment_count_with_node_max_sessions.md](./results_test_k8s_autoscaling_deployment_count_with_node_max_sessions.md)

# Resources

Expand Down
22 changes: 22 additions & 0 deletions .keda/results_test_k8s_autoscaling_deployment_count.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
| Iteration | New request sessions | Sessions created time | Sessions failed to create | New pods scaled up | Total running sessions | Total running pods | Max sessions per pod | Gaps | Sessions closed |
| --------- | -------------------- | --------------------- | ------------------------- | ------------------ | ---------------------- | ------------------ | -------------------- | ---- | --------------- |
| 1 | 1 | 39.44 s | 0 | 1 | 1 | 1 | 1 | 0 | 1 |
| 2 | 2 | 39.77 s | 0 | 1 | 2 | 2 | 1 | 0 | 0 |
| 3 | 1 | 28.38 s | 0 | 1 | 3 | 3 | 1 | 0 | 0 |
| 4 | 2 | 54.92 s | 0 | 2 | 5 | 5 | 1 | 0 | 0 |
| 5 | 2 | 33.04 s | 0 | 2 | 7 | 7 | 1 | 0 | 0 |
| 6 | 1 | 29.43 s | 0 | 1 | 8 | 8 | 1 | 0 | 8 |
| 7 | 2 | 47.04 s | 0 | 1 | 2 | 9 | 1 | 7 | 0 |
| 8 | 2 | 43.42 s | 0 | 1 | 4 | 10 | 1 | 6 | 0 |
| 9 | 2 | 47.99 s | 0 | 0 | 6 | 10 | 1 | 4 | 0 |
| 10 | 3 | 44.97 s | 0 | 2 | 9 | 12 | 1 | 3 | 0 |
| 11 | 3 | 42.98 s | 0 | -2 | 12 | 10 | 1 | -2 | 12 |
| 12 | 1 | 4.52 s | 0 | 0 | 1 | 10 | 1 | 9 | 0 |
| 13 | 2 | 39.93 s | 0 | 1 | 3 | 11 | 1 | 8 | 0 |
| 14 | 1 | 29.45 s | 0 | 1 | 4 | 12 | 1 | 8 | 0 |
| 15 | 2 | 30.52 s | 0 | 1 | 6 | 13 | 1 | 7 | 0 |
| 16 | 3 | 35.62 s | 0 | 3 | 9 | 16 | 1 | 7 | 9 |
| 17 | 3 | 16.59 s | 0 | 0 | 3 | 16 | 1 | 13 | 0 |
| 18 | 3 | 36.76 s | 0 | 1 | 6 | 10 | 1 | 4 | 0 |
| 19 | 3 | 29.10 s | 0 | 1 | 9 | 11 | 1 | 2 | 0 |
| 20 | 3 | 51.36 s | 0 | 3 | 12 | 14 | 1 | 2 | 0 |
Loading

0 comments on commit b52e28b

Please sign in to comment.