Benchmark runtime weights #146
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark runtime weights | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| rebuild-docker: | |
| type: boolean | |
| # if the runtime-benchmarks image should be rebuilt or pulled from hub | |
| description: rebuild-docker | |
| required: true | |
| default: true | |
| litentry: | |
| type: boolean | |
| description: litentry | |
| required: true | |
| default: true | |
| litmus: | |
| type: boolean | |
| description: litmus | |
| required: true | |
| default: true | |
| rococo: | |
| type: boolean | |
| description: rococo | |
| required: true | |
| default: true | |
| pallets: | |
| description: pallets to benchmark, * for all, or comma listed (e.g. frame-system,pallet-proxy) | |
| default: "*" | |
| required: true | |
| env: | |
| INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }} | |
| BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }} | |
| BENCHMARK_SSH_KEY: ${{ secrets.BENCHMARK_SSH_KEY }} | |
| DOCKER_BUILDKIT: 1 | |
| jobs: | |
| ## build docker image with runtime-benchmarks feature and push it to the hub | |
| build-docker: | |
| if: ${{ github.event.inputs.rebuild-docker == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Free up disk space | |
| if: startsWith(runner.name, 'GitHub Actions') | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| swap-storage: false | |
| large-packages: false | |
| - name: Checkout codes on ${{ github.ref }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # try to increase usable memory | |
| # https://github.com/actions/runner/issues/1051 | |
| - name: Set Swap Space | |
| uses: pierotofy/set-swap-space@master | |
| with: | |
| swap-size-gb: 10 | |
| - name: Build docker image | |
| run: | | |
| ./scripts/build-docker.sh production runtime-benchmarks --features=runtime-benchmarks | |
| - name: Dockerhub login | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Push docker image | |
| run: | | |
| docker push litentry/litentry-parachain:runtime-benchmarks | |
| ## run the benchmarking remotely | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| needs: build-docker | |
| # see https://github.com/actions/runner/issues/491 | |
| if: | | |
| always() && | |
| (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| steps: | |
| - name: Set env | |
| run: | | |
| chain="" | |
| if [ "${{ github.event.inputs.litmus }}" = "true" ]; then | |
| chain="$chain litmus" | |
| fi | |
| if [ "${{ github.event.inputs.litentry }}" = "true" ]; then | |
| chain="$chain litentry" | |
| fi | |
| if [ "${{ github.event.inputs.rococo }}" = "true" ]; then | |
| chain="$chain rococo" | |
| fi | |
| if [ "$chain" = "" ]; then | |
| echo "::error::Please select at least one chain." | |
| exit 1 | |
| fi | |
| echo "CHAIN=$chain" >> $GITHUB_ENV | |
| - name: Checkout codes on ${{ github.ref }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Pull docker image | |
| run: | | |
| docker pull litentry/litentry-parachain:runtime-benchmarks | |
| - name: Set up AWS CLI | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| # TODO: maybe use GHA to start/stop remote instance | |
| - name: Start remote instance | |
| timeout-minutes: 10 | |
| id: start_instance | |
| run: | | |
| aws ec2 start-instances --instance-ids ${{ env.INSTANCE_ID }} | |
| sleep 5 | |
| instance_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text" | |
| system_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text" | |
| SECONDS=0 | |
| while : ; do | |
| if [ "$(eval $instance_status)" = "ok" ] && [ "$(eval $system_status)" = "ok" ]; then | |
| break | |
| else | |
| sleep 20 | |
| SECONDS=$((SECONDS + 20)) | |
| fi | |
| done | |
| echo "Remote instance reachable now after $SECONDS seconds" | |
| remote_ip=`aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text` | |
| echo "Running instances ip address: $remote_ip" | |
| echo "remote_ip=$remote_ip" >> $GITHUB_OUTPUT | |
| # exit status should propagate through ssh | |
| - name: Remotely benchmark pallets ${{ github.event.inputs.pallets }} for ${{ env.CHAIN }} | |
| timeout-minutes: 240 | |
| run: | | |
| # prepend the asterisk with \ to go through ssh | |
| echo "Running instances ip address: ${{ steps.start_instance.outputs.remote_ip }}" | |
| arg="${{ github.event.inputs.pallets }}" | |
| chain="${{ env.CHAIN }}" | |
| if [ "$arg" = "*" ]; then | |
| arg="\\$arg"; | |
| fi | |
| cat << EOF > ./benchmark-server-key.pem | |
| ${{ env.BENCHMARK_SSH_KEY }} | |
| EOF | |
| chmod 600 ./benchmark-server-key.pem | |
| for c in $chain; do | |
| ssh -x -i ./benchmark-server-key.pem -o StrictHostKeychecking=no "${{ steps.start_instance.outputs.remote_ip }}" -l ${{ env.BENCHMARK_SSH_USER }} 'bash -s' < scripts/benchmark-weight-remote.sh "$c" "${GITHUB_REF#refs/heads/}" "$arg" | |
| echo "copy generated weights files back ..." | |
| scp -o StrictHostKeychecking=no -i ./benchmark-server-key.pem "${{ env.BENCHMARK_SSH_USER }}"@"${{ steps.start_instance.outputs.remote_ip }}":/tmp/litentry-parachain/runtime/$c/src/weights/*.rs runtime/$c/src/weights/ | |
| done | |
| rm -f ./benchmark-server-key.pem | |
| echo "======================" | |
| git status | |
| - name: Stop remote instance | |
| if: always() | |
| run: | | |
| aws ec2 stop-instances --instance-ids ${{ env.INSTANCE_ID }} | |
| sleep 5 | |
| ret=`aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'` | |
| echo "Remote instance running state: $ret" | |
| - name: Create auto PR | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| commit-message: "[benchmarking bot] Auto commit generated weights files" | |
| committer: benchmarking bot <[email protected]> | |
| author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> | |
| signoff: false | |
| branch: benchmarking-bot-${{ github.run_id }} | |
| delete-branch: true | |
| title: "[benchmarking bot] Update generated weights files" | |
| body: | | |
| This is an automatically created PR. | |
| It updates the weights files under `runtime/*/src/weights/*.rs` after running benchmarks on the remote machine: ${{ env.INSTANCE_ID }} | |
| Pallets: "${{ github.event.inputs.pallets }}" | |
| Chain: "${{ env.CHAIN }}" | |
| Github action run: https://github.com/litentry/litentry-parachain/actions/runs/${{ github.run_id }} | |
| labels: | | |
| automated-pr | |
| assignees: ${{ github.actor }} | |
| reviewers: ${{ github.actor }} | |
| draft: false |