Performance Test Suite #365
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Test Suite | |
| on: | |
| push: | |
| branches: | |
| - 'perf/**' | |
| schedule: | |
| - cron: '0 1 * * MON-FRI' | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| required: false | |
| env: | |
| TF_AWS_REGION: eu-west-1 | |
| TF_VAR_s3_bucket_name: tf-emqx-performance-test2 | |
| TF_VAR_test_duration: 1800 | |
| TF_VAR_prometheus_remote_write_region: eu-west-1 | |
| TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }} | |
| permissions: | |
| contents: read | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-latest | |
| if: github.repository_owner == 'emqx' | |
| container: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu20.04 | |
| outputs: | |
| BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} | |
| PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} | |
| steps: | |
| - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Work around https://github.com/actions/checkout/issues/766 | |
| run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - id: prepare | |
| run: | | |
| echo "EMQX_NAME=emqx" >> $GITHUB_ENV | |
| echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV | |
| echo "BENCH_ID=$(date --utc +%F)/emqx-$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT | |
| - name: Build deb package | |
| run: | | |
| make ${EMQX_NAME}-pkg | |
| ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg | |
| - name: Get package file name | |
| id: package_file | |
| run: | | |
| echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| with: | |
| name: emqx-ubuntu20.04 | |
| path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }} | |
| scenario_1on1: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| env: | |
| TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }} | |
| aws-region: eu-west-1 | |
| - name: Checkout tf-emqx-performance-test | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: emqx/tf-emqx-performance-test | |
| path: tf-emqx-performance-test | |
| ref: v0.2.3 | |
| - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 | |
| with: | |
| name: emqx-ubuntu20.04 | |
| path: tf-emqx-performance-test/ | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 | |
| with: | |
| terraform_wrapper: false | |
| - name: run scenario | |
| working-directory: ./tf-emqx-performance-test | |
| timeout-minutes: 60 | |
| env: | |
| TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1on1" | |
| TF_VAR_use_emqttb: 1 | |
| TF_VAR_use_emqtt_bench: 0 | |
| TF_VAR_emqttb_instance_count: 1 | |
| TF_VAR_emqttb_instance_type: c5.2xlarge | |
| TF_VAR_emqttb_scenario: '@pubsub_fwd -n 50_000 --pub-qos 1 --sub-qos 1' | |
| TF_VAR_emqx_instance_type: c5.2xlarge | |
| TF_VAR_emqx_instance_count: 3 | |
| run: | | |
| terraform init | |
| terraform apply -auto-approve | |
| ./wait-emqttb.sh | |
| ./fetch-metrics.sh | |
| terraform destroy -auto-approve | |
| aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id . | |
| - name: Send notification to Slack | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 | |
| with: | |
| payload-file-path: "./tf-emqx-performance-test/slack-payload.json" | |
| - name: terraform destroy | |
| if: always() | |
| working-directory: ./tf-emqx-performance-test | |
| run: | | |
| terraform destroy -auto-approve | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: success() | |
| with: | |
| name: metrics | |
| path: | | |
| "./tf-emqx-performance-test/*.tar.gz" | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: failure() | |
| with: | |
| name: terraform | |
| path: | | |
| ./tf-emqx-performance-test/.terraform | |
| ./tf-emqx-performance-test/*.tfstate | |
| scenario_fanout: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| - scenario_1on1 | |
| env: | |
| TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }} | |
| aws-region: eu-west-1 | |
| - name: Checkout tf-emqx-performance-test | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: emqx/tf-emqx-performance-test | |
| path: tf-emqx-performance-test | |
| ref: v0.2.3 | |
| - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 | |
| with: | |
| name: emqx-ubuntu20.04 | |
| path: tf-emqx-performance-test/ | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 | |
| with: | |
| terraform_wrapper: false | |
| - name: run scenario | |
| working-directory: ./tf-emqx-performance-test | |
| timeout-minutes: 60 | |
| env: | |
| TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/fan-out" | |
| TF_VAR_use_emqttb: 1 | |
| TF_VAR_use_emqtt_bench: 0 | |
| TF_VAR_emqttb_instance_count: 1 | |
| TF_VAR_emqttb_instance_type: c5.2xlarge | |
| TF_VAR_emqttb_scenario: '@pub --topic "t/%n" --conninterval 10ms --pubinterval 20ms --num-clients 5 --size 16 @sub --topic "t/#" --conninterval 10ms --num-clients 1000' | |
| TF_VAR_emqx_instance_type: c5.large | |
| TF_VAR_emqx_instance_count: 3 | |
| run: | | |
| terraform init | |
| terraform apply -auto-approve | |
| ./wait-emqttb.sh | |
| ./fetch-metrics.sh | |
| terraform destroy -auto-approve | |
| aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id . | |
| - name: Send notification to Slack | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 | |
| with: | |
| payload-file-path: "./tf-emqx-performance-test/slack-payload.json" | |
| - name: terraform destroy | |
| if: always() | |
| working-directory: ./tf-emqx-performance-test | |
| run: | | |
| terraform destroy -auto-approve | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: success() | |
| with: | |
| name: metrics | |
| path: | | |
| "./tf-emqx-performance-test/*.tar.gz" | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: failure() | |
| with: | |
| name: terraform | |
| path: | | |
| ./tf-emqx-performance-test/.terraform | |
| ./tf-emqx-performance-test/*.tfstate | |
| scenario_fanin: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| - scenario_1on1 | |
| - scenario_fanout | |
| env: | |
| TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }} | |
| aws-region: eu-west-1 | |
| - name: Checkout tf-emqx-performance-test | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: emqx/tf-emqx-performance-test | |
| path: tf-emqx-performance-test | |
| ref: v0.2.3 | |
| - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 | |
| with: | |
| name: emqx-ubuntu20.04 | |
| path: tf-emqx-performance-test/ | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 | |
| with: | |
| terraform_wrapper: false | |
| - name: run scenario | |
| working-directory: ./tf-emqx-performance-test | |
| timeout-minutes: 60 | |
| env: | |
| TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/fan-in" | |
| TF_VAR_use_emqttb: 1 | |
| TF_VAR_use_emqtt_bench: 0 | |
| TF_VAR_emqttb_instance_count: 2 | |
| TF_VAR_emqttb_start_n_multiplier: 25000 | |
| TF_VAR_emqttb_instance_type: c5.xlarge | |
| TF_VAR_emqttb_scenario: '@pub --topic t/%n --conninterval 10ms --pubinterval 1s --num-clients 25_000 --start-n $START_N --size 16 @sub --topic \$share/perf/t/# --conninterval 10ms --num-clients 250' | |
| TF_VAR_emqx_instance_type: c5.2xlarge | |
| TF_VAR_emqx_instance_count: 3 | |
| run: | | |
| terraform init | |
| terraform apply -auto-approve | |
| ./wait-emqttb.sh | |
| ./fetch-metrics.sh | |
| terraform destroy -auto-approve | |
| aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id . | |
| - name: Send notification to Slack | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 | |
| with: | |
| payload-file-path: "./tf-emqx-performance-test/slack-payload.json" | |
| - name: terraform destroy | |
| if: always() | |
| working-directory: ./tf-emqx-performance-test | |
| run: | | |
| terraform destroy -auto-approve | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: success() | |
| with: | |
| name: metrics | |
| path: | | |
| "./tf-emqx-performance-test/*.tar.gz" | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: failure() | |
| with: | |
| name: terraform | |
| path: | | |
| ./tf-emqx-performance-test/.terraform | |
| ./tf-emqx-performance-test/*.tfstate | |
| scenario_1m_conns: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| - scenario_fanin | |
| - scenario_fanout | |
| - scenario_1on1 | |
| env: | |
| TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }} | |
| aws-region: eu-west-1 | |
| - name: Checkout tf-emqx-performance-test | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| repository: emqx/tf-emqx-performance-test | |
| path: tf-emqx-performance-test | |
| ref: v0.2.3 | |
| - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 | |
| with: | |
| name: emqx-ubuntu20.04 | |
| path: tf-emqx-performance-test/ | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0 | |
| with: | |
| terraform_wrapper: false | |
| - name: run scenario | |
| working-directory: ./tf-emqx-performance-test | |
| timeout-minutes: 60 | |
| env: | |
| TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1m-connections" | |
| TF_VAR_use_emqttb: 1 | |
| TF_VAR_use_emqtt_bench: 0 | |
| TF_VAR_emqttb_instance_count: 5 | |
| TF_VAR_emqttb_instance_type: c5.2xlarge | |
| TF_VAR_emqttb_scenario: '@conn -N 200_000 --conninterval 1ms' | |
| TF_VAR_emqx_instance_type: c5.2xlarge | |
| TF_VAR_emqx_instance_count: 3 | |
| run: | | |
| terraform init | |
| terraform apply -auto-approve | |
| ./wait-emqttb.sh | |
| ./fetch-metrics.sh | |
| terraform destroy -auto-approve | |
| aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id . | |
| - name: Send notification to Slack | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 | |
| with: | |
| payload-file-path: "./tf-emqx-performance-test/slack-payload.json" | |
| - name: terraform destroy | |
| if: always() | |
| working-directory: ./tf-emqx-performance-test | |
| run: | | |
| terraform destroy -auto-approve | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: success() | |
| with: | |
| name: metrics | |
| path: | | |
| "./tf-emqx-performance-test/*.tar.gz" | |
| - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: failure() | |
| with: | |
| name: terraform | |
| path: | | |
| ./tf-emqx-performance-test/.terraform | |
| ./tf-emqx-performance-test/*.tfstate |