Node Throughput Test #51
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Node Throughput Test | |
| on: | |
| schedule: | |
| - cron: '39 0 * * *' | |
| permissions: | |
| # To be able to access the repository with actions/checkout | |
| contents: read | |
| concurrency: | |
| # Structure: | |
| # - Workflow name | |
| # - Event type | |
| # - A unique identifier depending on event type: | |
| # - schedule: SHA | |
| # - workflow_dispatch: PR number | |
| # | |
| # This structure ensures a unique concurrency group name is generated for each | |
| # type of testing, such that re-runs will cancel the previous run. | |
| group: | | |
| ${{ github.workflow }} | |
| ${{ github.event_name }} | |
| ${{ | |
| (github.event_name == 'schedule' && github.sha) || | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number) | |
| }} | |
| cancel-in-progress: true | |
| env: | |
| # renovate: datasource=github-releases depName=kubernetes/kops | |
| kops_version: v1.28.1 | |
| # renovate: datasource=golang-version depName=go | |
| go_version: 1.21.5 | |
| # Adding k8s.local to the end makes kops happy- | |
| # has stricter DNS naming requirements. | |
| test_name: node-throughput | |
| cluster_base_name: ${{ github.run_id }}-${{ github.run_attempt }}.k8s.local | |
| GCP_PERF_RESULTS_BUCKET: gs://cilium-scale-results | |
| CILIUM_CLI_MODE: helm | |
| jobs: | |
| install-and-scaletest: | |
| runs-on: ubuntu-latest | |
| name: Install and Scale Test | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Checkout context ref (trusted) | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| ref: ${{ inputs.context-ref || github.sha }} | |
| persist-credentials: false | |
| - name: Set Environment Variables | |
| uses: ./.github/actions/set-env-variables | |
| - name: Get Cilium's default values | |
| id: default_vars | |
| uses: ./.github/actions/helm-default | |
| with: | |
| image-tag: ${{ github.sha }} | |
| - name: Set up job variables | |
| id: vars | |
| run: | | |
| SHA="${{ github.sha }}" | |
| # Setup Cilium install options | |
| CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \ | |
| --wait=false" | |
| CLUSTER_NAME="${{ env.test_name }}-${{ env.cluster_base_name }}" | |
| echo SHA=${SHA} >> $GITHUB_OUTPUT | |
| echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT | |
| echo CLUSTER_NAME=${CLUSTER_NAME} >> $GITHUB_OUTPUT | |
| - name: Wait for images to be available | |
| timeout-minutes: 30 | |
| shell: bash | |
| run: | | |
| for image in cilium-ci operator-generic-ci hubble-relay-ci ; do | |
| until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.SHA }} &> /dev/null; do sleep 45s; done | |
| done | |
| - name: Install Go | |
| uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 | |
| with: | |
| go-version: ${{ env.go_version }} | |
| - name: Install Cilium CLI | |
| uses: cilium/cilium-cli@c1fa8afda634f0457e9905d4f638e5164c4e0fc8 # v0.15.17 | |
| with: | |
| repository: ${{ env.CILIUM_CLI_RELEASE_REPO }} | |
| release-version: ${{ env.CILIUM_CLI_VERSION }} | |
| - name: Install Kops | |
| uses: cilium/scale-tests-action/install-kops@8a522b9d71254b6f6615c296e41a70610c9615ea # main | |
| with: | |
| release_version: ${{ env.kops_version }} | |
| - name: Setup gcloud credentials | |
| uses: google-github-actions/auth@67e9c72af6e0492df856527b474995862b7b6591 # v2.0.0 | |
| with: | |
| credentials_json: '${{ secrets.GCP_PERF_SA_KEY }}' | |
| - name: Setup gcloud CLI | |
| uses: google-github-actions/setup-gcloud@825196879a077b7efa50db2e88409f44de4635c2 # v2.0.0 | |
| with: | |
| project_id: ${{ secrets.GCP_PERF_PROJECT_ID }} | |
| version: "405.0.0" | |
| - name: Clone ClusterLoader2 | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| repository: cilium/perf-tests | |
| ref: 6f1bf283e22aa4c29d5db2a7b52501b091a0912e # master | |
| persist-credentials: false | |
| sparse-checkout: clusterloader2 | |
| path: perf-tests | |
| - name: Display version info of installed tools | |
| run: | | |
| echo "--- go ---" | |
| go version | |
| echo "--- cilium-cli ---" | |
| cilium version --client | |
| echo "--- kops ---" | |
| ./kops version | |
| echo "--- gcloud ---" | |
| gcloud version | |
| - name: Deploy cluster | |
| id: deploy-cluster | |
| uses: cilium/scale-tests-action/create-cluster@8a522b9d71254b6f6615c296e41a70610c9615ea # main | |
| timeout-minutes: 30 | |
| with: | |
| cluster_name: ${{ steps.vars.outputs.cluster_name }} | |
| control_plane_size: n2-standard-4 | |
| control_plane_count: 1 | |
| node_size: e2-standard-8 | |
| node_count: 1 | |
| kops_state: ${{ secrets.GCP_PERF_KOPS_STATE_STORE }} | |
| project_id: ${{ secrets.GCP_PERF_PROJECT_ID }} | |
| - name: Setup firewall rules | |
| uses: cilium/scale-tests-action/setup-firewall@8a522b9d71254b6f6615c296e41a70610c9615ea # main | |
| with: | |
| cluster_name: ${{ steps.vars.outputs.cluster_name }} | |
| - name: Install Cilium | |
| run: | | |
| cilium install ${{ steps.vars.outputs.cilium_install_defaults }} | |
| - name: Wait for cluster to be ready | |
| uses: cilium/scale-tests-action/validate-cluster@8a522b9d71254b6f6615c296e41a70610c9615ea # main | |
| timeout-minutes: 20 | |
| with: | |
| cluster_name: ${{ steps.vars.outputs.cluster_name }} | |
| kops_state: ${{ secrets.GCP_PERF_KOPS_STATE_STORE }} | |
| - name: Wait for Cilium status to be ready | |
| run: | | |
| cilium status --wait | |
| - name: Run CL2 | |
| id: run-cl2 | |
| working-directory: ./perf-tests/clusterloader2 | |
| timeout-minutes: 30 | |
| shell: bash | |
| # --enable-exec-service=false to reduce number of pods so 100 pods can fit in node | |
| # POD_STARTUP_LATENCY_THRESHOLD=60s so the test doesn't fail, currently we have ~30s pods startup latency | |
| run: | | |
| mkdir ./report | |
| echo POD_STARTUP_LATENCY_THRESHOLD: 60s >> ./testoverrides.yaml | |
| go run ./cmd/clusterloader.go \ | |
| -v=4 \ | |
| --testconfig=./testing/node-throughput/config.yaml \ | |
| --testoverrides=./testoverrides.yaml \ | |
| --enable-exec-service=false \ | |
| --provider=gce \ | |
| --enable-prometheus-server \ | |
| --tear-down-prometheus-server=false \ | |
| --report-dir=./report \ | |
| --kubeconfig=$HOME/.kube/config \ | |
| 2>&1 | tee cl2-output.txt | |
| - name: Get sysdump | |
| if: ${{ always() && steps.run-cl2.outcome != 'skipped' }} | |
| run: | | |
| cilium status | |
| cilium sysdump --output-filename cilium-sysdump-final | |
| - name: Cleanup cluster | |
| if: ${{ always() && steps.deploy-cluster.outcome != 'skipped' }} | |
| uses: cilium/scale-tests-action/cleanup-cluster@8a522b9d71254b6f6615c296e41a70610c9615ea # main | |
| with: | |
| cluster_name: ${{ steps.vars.outputs.cluster_name }} | |
| kops_state: ${{ secrets.GCP_PERF_KOPS_STATE_STORE }} | |
| - name: Export results and sysdump to GS bucket | |
| if: ${{ always() && steps.run-cl2.outcome != 'skipped' }} | |
| uses: cilium/scale-tests-action/export-results@8a522b9d71254b6f6615c296e41a70610c9615ea # main | |
| with: | |
| test_name: ${{ env.test_name }} | |
| results_bucket: ${{ env.GCP_PERF_RESULTS_BUCKET }} | |
| artifacts: ./perf-tests/clusterloader2/report/* | |
| other_files: cilium-sysdump-final.zip ./perf-tests/clusterloader2/cl2-output.txt |